<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="354" failures="1" skipped="82">
	<testsuite name="" tests="354" failures="1" errors="0" id="0" hostname="e2e-cli-nmanos-nmanos-konfld0db57b5230a9ee5fa1baef028b5b4a5-pod" skipped="82" time="7657.310" timestamp="2026-09-18T13:16:37Z">
		<testcase name="TestOdhOperator" classname="" time="1020.860">
			<system-out><![CDATA[2026/09/18 12:59:36 ARTIFACT_DIR not set, skipping periodic metrics collection
2026-09-18T12:59:37Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests" classname="" time="0.240">
			<system-out><![CDATA[2026-09-18T12:59:37Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Datascience_Cluster_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataScienceCluster_Initialization_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_FeatureTracker_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Ray_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_ModelRegistry_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrustyAI_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Kueue_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrainingOperator_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Trainer_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataSciencePipelines_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Monitoring_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Auth_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_SparkOperator_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Platform_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests" classname="" time="0.650">
			<system-out><![CDATA[2026-09-18T12:59:37Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_HardwareProfile_resource" classname="" time="10.190"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_owned_namespaces_exist" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_default_NetworkPolicy_exist" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_denies_invalid_PEM_on_update" classname="" time="0.070"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_valid_PEM_on_update" classname="" time="0.200"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_invalid_PEM_when_Removed" classname="" time="0.190"></testcase>
		<testcase name="TestOdhOperator/monitoring" classname="" time="0.210">
			<skipped message="Skipped"><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}
    controller_test.go:767: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components" classname="" time="1007.600"></testcase>
		<testcase name="TestOdhOperator/components/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway" classname="" time="61.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard" classname="" time="202.740">
			<system-out><![CDATA[2026-09-18T13:00:49Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines" classname="" time="96.810">
			<system-out><![CDATA[2026-09-18T13:00:09Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator" classname="" time="20.920"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve" classname="" time="339.430"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator" classname="" time="71.670"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx" classname="" time="91.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray" classname="" time="101.670">
			<system-out><![CDATA[2026-09-18T13:00:50Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer" classname="" time="112.240">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator" classname="" time="97.000"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches" classname="" time="61.970"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_enabled" classname="" time="51.100">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_enabled" classname="" time="50.670">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_enabled" classname="" time="70.670">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_enabled" classname="" time="51.170">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_component_enabled" classname="" time="30.780">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_enabled" classname="" time="71.750">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_enabled" classname="" time="40.970">
			<system-out><![CDATA[2026-09-18T12:59:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade_from_in-tree:_selector_migration" classname="" time="0.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_component_enabled" classname="" time="0.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_operator_deployed" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_created" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_ready" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_feast-operator_deployed_by_module" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade:_existing_operands_preserved" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_disabled_cleanup" classname="" time="20.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_enabled" classname="" time="30.640"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_update_operand_resources" classname="" time="20.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_update_operand_resources/deployment_kubeflow-training-operator" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_update_operand_resources" classname="" time="20.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_update_operand_resources/deployment_kubeflow-trainer-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_with_mixed_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_disabled" classname="" time="10.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_component_releases" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_operator_deployment" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery" classname="" time="35.450"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="5.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_workbenches_namespace_configuration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kubeflow-rhoai-config-82t856tcf7" classname="" time="5.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_ImageStreams_available" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_MLflow_integration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery" classname="" time="0.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_disabled" classname="" time="20.260">
			<system-out><![CDATA[    helper_test.go:157: No ConfigMap resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.040">
			<system-out><![CDATA[    helper_test.go:157: No Service resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: No Deployment resources found for component Workbenches, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_disabled" classname="" time="10.480"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_conditions" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_operands_have_OwnerReferences" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources" classname="" time="20.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources/deployment_data-science-pipelines-operator-controller-manager" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="5.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kubeflow-training-operator" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_component_enabled" classname="" time="60.900"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_releases" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery" classname="" time="40.510"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kubeflow-trainer-config" classname="" time="5.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="25.190"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kubeflow-training-operator" classname="" time="25.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_enabled" classname="" time="40.590"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_rhoai-config-7t92dcgb54" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_disabled" classname="" time="20.200"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="5.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kubeflow-trainer-controller-manager" classname="" time="5.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_argoWorkflowsControllers_options" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery" classname="" time="35.490"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="15.260"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-config" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_spec" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_no_Kserve_FeatureTrackers" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_VAP_created_when_kserve_is_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources" classname="" time="80.200"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="25.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kubeflow-trainer-controller-manager" classname="" time="25.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-parameters" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_workflow-controller-configmap" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainingoperator/Validate_component_disabled" classname="" time="10.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="5.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_data-science-pipelines-operator-service" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_llmisvc-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_data-science-pipelines-operator-controller-manager" classname="" time="15.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources" classname="" time="20.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources/deployment_kuberay-operator" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_disabled" classname="" time="10.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_model-serving-api" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_operands_have_OwnerReferences" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_update_operand_resources" classname="" time="20.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery" classname="" time="30.400"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_ray-config-gb8c2tkkcg" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="10.190"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-operator" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_odh-model-controller" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-webhook-service" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kuberay-operator" classname="" time="15.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_dynamically_watches_operands" classname="" time="10.190"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_releases" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_well-known_LLMInferenceServiceConfig_versioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_subscription_dependency_conditions" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_platform_config_ConfigMap" classname="" time="0.030">
			<system-out><![CDATA[    helper_test.go:157: Verifying platform config ConfigMap odh-kserve-config exists in namespace redhat-ods-applications.
    helper_test.go:157: Non-XKS platform: verifying cert-manager CA keys are absent.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_ModelRegistry_state_propagation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery" classname="" time="176.740"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="15.320"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_inferenceservice-config" classname="" time="5.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_CRDs_reinstated" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_VAP_blocks_dashboard_HardwareProfile_and_AcceleratorProfile_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery" classname="" time="111.300"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-parameters" classname="" time="5.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="100.670"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_odh-model-controller-parameters" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_agent-ops-params" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="40.770"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_automl-params" classname="" time="95.060">
			<failure message="Failed"></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-metrics-service" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-service" classname="" time="5.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-webhook-server-service" classname="" time="5.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-controller-manager-service" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-webhook-server-service" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_model-serving-api" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-metrics-service" classname="" time="5.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-webhook-service" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="120.640"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-controller-manager" classname="" time="45.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_llmisvc-controller-manager" classname="" time="45.200">
			<system-out><![CDATA[    test_context_test.go:1326: 
        Timed out after 90.003s.
        Resource ConfigMap redhat-ods-applications/automl-params was not properly recreated with new identity
        The function passed to Eventually failed at /workspace/tests/e2e/test_context_test.go:1307 with:
        Expected ConfigMap redhat-ods-applications/automl-params to be recreated
        Expected
            <*unstructured.Unstructured | 0x0>: nil
        not to be nil
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_model-serving-api" classname="" time="15.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_odh-model-controller" classname="" time="15.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_disabled" classname="" time="10.270"></testcase>
		<testcase name="TestOdhOperator/components/group_2" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue" classname="" time="103.880"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry" classname="" time="91.670">
			<system-out><![CDATA[2026-09-18T13:05:28Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_error_with_ocp_kueue-operator_not_installed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[2026-09-18T13:05:28Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}
    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_removed_to_unmanaged_transition" classname="" time="21.200"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_enabled" classname="" time="30.560">
			<system-out><![CDATA[    helper_test.go:157: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:157: Cleaning up any existing Kueue test resources.
    helper_test.go:157: Cleaning up Kueue cluster-scoped resources
    helper_test.go:157: Will attempt to delete 3 Kueue resources
    helper_test.go:157: Attempting to delete ClusterQueue /default
    helper_test.go:157: Successfully processed deletion of ClusterQueue /default
    helper_test.go:157: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:157: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:157: Attempting to delete Kueue /cluster
    helper_test.go:157: Successfully processed deletion of Kueue /cluster
    helper_test.go:157: Creating test namespace (test-kueue-managed-damje67hk0ss7mvinqgg) with Kueue management annotation.
    helper_test.go:157: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:157: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:157: Updating the management state of the component in the DataScienceCluster to Unmanaged with autoCreateQueues=true (expecting NotReady initially due to missing operator).
    helper_test.go:157: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:157: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:157: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:157: Verifying the component in the DataScienceCluster transitions to Ready state.
    helper_test.go:157: Verifying Kueue configuration (cluster) is created with all expected frameworks.
    helper_test.go:157: Verifying default ClusterQueue and LocalQueue resources are created in namespace test-kueue-managed-damje67hk0ss7mvinqgg.
    helper_test.go:157: Verifying default resource flavor (default-flavor) is created.
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind']]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_to_removed_transition" classname="" time="51.200">
			<system-out><![CDATA[    helper_test.go:157: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_spec" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_update_operand_resources" classname="" time="20.150"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_update_operand_resources/deployment_model-registry-operator-controller-manager" classname="" time="20.050">
			<system-out><![CDATA[    helper_test.go:157: Cleaning up any existing Kueue test resources.
    helper_test.go:157: Cleaning up Kueue cluster-scoped resources
    helper_test.go:157: Will attempt to delete 3 Kueue resources
    helper_test.go:157: Attempting to delete ClusterQueue /default
    helper_test.go:157: Successfully processed deletion of ClusterQueue /default
    helper_test.go:157: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:157: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:157: Attempting to delete Kueue /cluster
    helper_test.go:157: Successfully processed deletion of Kueue /cluster
    helper_test.go:157: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:157: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:157: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:157: Defining expected conditions for Unmanaged state (Ready=True).
    helper_test.go:157: Updating DSC default-dsc to set Kueue managementState to Unmanaged with autoCreateQueues=true.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_CRDs_reinstated" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_releases" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery" classname="" time="30.440"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="5.100"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_model-registry-operator-parameters" classname="" time="5.080">
			<system-out><![CDATA[    helper_test.go:157: Creating test namespace (test-kueue-managed-damjebfhk0ss7mvinqh0) with Kueue management annotation.
    helper_test.go:157: Verifying default Kueue resources exist in namespace test-kueue-managed-damjebfhk0ss7mvinqh0.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="10.200"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_model-registry-operator-controller-manager-metrics-service" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_model-registry-operator-webhook-service" classname="" time="5.090">
			<system-out><![CDATA[    helper_test.go:157: Verifying default resource flavor (default-flavor) is created.
    helper_test.go:157: Updating DSC default-dsc to set Kueue managementState to Removed.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.140"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_model-registry-operator-controller-manager" classname="" time="15.120">
			<system-out><![CDATA[    helper_test.go:157: Verifying default resources are still present in namespace test-kueue-managed-damjebfhk0ss7mvinqh0.
    helper_test.go:157: Cleaning up Kueue test resources.
    helper_test.go:157: Cleaning up Kueue cluster-scoped resources
    helper_test.go:157: Will attempt to delete 3 Kueue resources
    helper_test.go:157: Attempting to delete ClusterQueue /default
    helper_test.go:157: Successfully processed deletion of ClusterQueue /default
    helper_test.go:157: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:157: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:157: Attempting to delete Kueue /cluster
    helper_test.go:157: Successfully processed deletion of Kueue /cluster]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_autoCreateQueues_disabled_skips_queue_creation" classname="" time="21.020">
			<system-out><![CDATA[    helper_test.go:157: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:157: Cleaning up any existing Kueue test resources.
    helper_test.go:157: Cleaning up Kueue cluster-scoped resources
    helper_test.go:157: Will attempt to delete 3 Kueue resources
    helper_test.go:157: Attempting to delete ClusterQueue /default
    helper_test.go:157: Successfully processed deletion of ClusterQueue /default
    helper_test.go:157: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:157: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:157: Attempting to delete Kueue /cluster
    helper_test.go:157: Successfully processed deletion of Kueue /cluster
    helper_test.go:157: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:157: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_disabled" classname="" time="10.220">
			<system-out><![CDATA[    helper_test.go:157: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:157: Creating test namespace (test-kueue-no-queues-damjeo7hk0ss7mvinqhg) with Kueue management annotation.
    helper_test.go:157: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:157: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:157: Updating DSC to set Kueue to Unmanaged with autoCreateQueues=false (default).]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: No ConfigMap resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: No Service resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: No Deployment resources found for component Kueue, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_disabled" classname="" time="10.210">
			<system-out><![CDATA[    helper_test.go:157: Setting DSC component default-kueue to Removed state.
    helper_test.go:157: Verifying no Deployments remain in namespace redhat-ods-applications with part-of=kueue label.
    helper_test.go:157: Ensuring the component CR /default-kueue is removed.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator" classname="" time="101.640"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai" classname="" time="299.660">
			<system-out><![CDATA[2026-09-18T13:07:12Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_enabled" classname="" time="121.590">
			<system-out><![CDATA[2026-09-18T13:07:12Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_enabled" classname="" time="91.110"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_disabled" classname="" time="10.350"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_update_operand_resources" classname="" time="20.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_update_operand_resources/deployment_trustyai-service-operator-controller-manager" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_releases" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_MCP_guardrails_mode" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_pre_check" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery" classname="" time="137.260"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="106.840"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-dsc-config" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-config" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-coding-v1" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-instruction-following-v1" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-leaderboard-v2" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-long-context-v1" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-model-validation" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-reasoning-v1" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-safety-and-fairness-v1" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-standard-llm-evals-v1" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-toxicity-and-ethical-principles" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-deepeval" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-garak" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-garak-kfp" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-guidellm" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-ibm-clear" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-inspect" classname="" time="5.110"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-lighteval" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-lm-evaluation-harness" classname="" time="5.160"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-ragas" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-lmeval-tasks" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="15.280"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-controller-manager-metrics-service" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-metrics-service" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-webhook-service" classname="" time="5.090"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.140"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_trustyai-service-operator-controller-manager" classname="" time="15.120"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_disabled" classname="" time="20.480"></testcase>
		<testcase name="TestOdhOperator/components/group_4" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_4/modelsasservice" classname="" time="71.450">
			<system-out><![CDATA[2026-09-18T13:12:11Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}
    controller_test.go:767: Creating MaaS GatewayClass and Gateway: openshift-ingress/maas-default-gateway
    controller_test.go:767: Using hostname=maas.apps.rosa.nmanos-konflux1.durx.p3.openshiftapps.com, certSecret=2r2qdb90ltvhjc43lp6iqc6aqb7bgjda-primary-cert-bundle-secret
    controller_test.go:767: MaaS Gateway openshift-ingress/maas-default-gateway created with HTTP+HTTPS listeners
    controller_test.go:767: Creating MaaS PostgreSQL instance in namespace redhat-ai-gateway-infra
    controller_test.go:767: MaaS PostgreSQL instance and maas-db-config secret created in namespace redhat-ai-gateway-infra]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/modelsasservice/Validate_MaasTenantConfig_CR_in_subscription_namespace" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: Checking MaasTenantConfig models-as-a-service/default-tenant exists with DependenciesAvailable=True]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/modelsasservice/Validate_MaasTenantConfig_CRD_is_namespace-scoped" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: Checking CRD maastenantconfigs.maas.opendatahub.io has scope: Namespaced]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/modelsasservice/Validate_MaasTenantConfig_singleton_enforcement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/modelsasservice/Validate_payload-processing_egress_NetworkPolicy" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/modelsasservice/Validate_MaaS_deployment_removed_on_disable" classname="" time="20.140">
			<system-out><![CDATA[    helper_test.go:157: Disabling MaaS subcomponent (modelsAsAService=Removed)
    helper_test.go:157: Waiting until maas-controller Deployment is removed from redhat-ods-applications]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve" classname="" time="91.350"></testcase>
		<testcase name="TestOdhOperator/components/group_5/kueue" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:767: Skipping: flaky due to namespace stuck in Terminating state. See RHOAIENG-46773 and RHOAIENG-46774.]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/trainer" classname="" time="30.760"></testcase>
		<testcase name="TestOdhOperator/components/group_5/trainer/Validate_component_enabled" classname="" time="30.530">
			<system-out><![CDATA[2026-09-18T13:13:23Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve/Validate_component_enabled" classname="" time="91.120"></testcase>
		<testcase name="TestOdhOperator/components/group_5/trainer/Validate_external_operator_degraded_condition_monitoring" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve/Validate_external_operator_degraded_condition_monitoring" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_6" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_6/kserve-modelcache" classname="" time="101.390">
			<system-out><![CDATA[2026-09-18T13:14:54Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_6/kserve-modelcache/Validate_component_enabled" classname="" time="101.210"></testcase>
		<testcase name="TestOdhOperator/components/group_6/kserve-modelcache/Validate_ModelCache_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_6/kserve-modelcache/Validate_ModelCache_ConfigMap" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_6/kserve-modelcache/Validate_ModelCache_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_6/kserve-modelcache/Validate_ModelCache_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services" classname="" time="0.310"></testcase>
		<testcase name="TestOdhOperator/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/services/group_1/auth" classname="" time="0.310"></testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway" classname="" time="0.220">
			<system-out><![CDATA[2026-09-18T13:16:36Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_Auth_system_initialization" classname="" time="0.130">
			<system-out><![CDATA[2026-09-18T13:16:36Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_GatewayConfig_creation" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: Validating GatewayConfig resource
    helper_test.go:157: GatewayConfig validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_infrastructure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_client_and_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_proxy_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_token_forwarding_to_dashboard" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_kube-auth-proxy_TLS_args_match_cluster_APIServer" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_HorizontalPodAutoscaler_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_NetworkPolicy_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_callback_HTTPRoute" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EnvoyFilter_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EDS_endpoint_discovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_ready_status" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:157: Validating Gateway ready status
    helper_test.go:157: Gateway ready status validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_dashboard_redirect_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_addition_of_RoleBinding_when_group_is_added" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_removal_of_bindings_when_a_group_is_removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_blocks_Auth_update_with_invalid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_allows_Auth_with_valid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests" classname="" time="0.170">
			<system-out><![CDATA[2026-09-18T13:16:36Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_operator_deployment_health" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_leader_election_behavior" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_success" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_failure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_missing_CRD_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_RBAC_restriction_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/V2_to_V3_upgrade_E2E_Tests" classname="" time="0.170">
			<skipped message="Skipped"><![CDATA[2026-09-18T13:16:37Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}
    controller_test.go:767: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests" classname="" time="0.180">
			<system-out><![CDATA[2026-09-18T13:16:37Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_creation_of_configmap_with_deletion_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_that_owned_namespaces_are_not_deleted" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:157: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/upgrade_DSC_and_DSCI_v1_API" classname="" time="0.170">
			<skipped message="Skipped"><![CDATA[2026-09-18T13:16:37Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}
    controller_test.go:767: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestLogPodsSectionContinuesWithPartialData" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/all_enabled_when_no_flags" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/explicit_inclusion_limits_to_named_tests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/exclusion_removes_from_full_set" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/inclusion_and_exclusion_combined" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test/monitoring" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:78: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:85: Test nonexistent is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:94: Test monitoring is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/auth" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/gateway" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:128: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:142: No tests remaining in group single after excluding only]]></skipped>
		</testcase>
		<testcase name="TestIsTestEnabled" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/enabled_by_default" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/disabled_by_exclusion_flag" classname="" time="0.000"></testcase>
		<system-out><![CDATA[    controller_test.go:767: Skipping tests for components/sparkoperator
2026/09/18 13:04:11 TEST FAILURE DETECTED: ConfigMap deletion recovery
2026/09/18 13:04:11 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:ConfigMap deletion recovery) ===
2026/09/18 13:04:11 === CLUSTER STATE ===
2026/09/18 13:04:11 Node: ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:04:11   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:04:11   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:04:11   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:04:11   Resources: 15500m CPU, 63871024Ki memory allocatable (of 16000m CPU, 65022000Ki memory capacity)
2026/09/18 13:04:11 Node: ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:04:11   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:04:11   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:04:11   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:04:11   Resources: 15500m CPU, 63182884Ki memory allocatable (of 16000m CPU, 64333860Ki memory capacity)
2026/09/18 13:04:11 === DEPLOYMENTS ===
2026/09/18 13:04:11 Failed to collect deployment data: deployments not ready: redhat-ods-applications/llmisvc-controller-manager (0/1)
2026/09/18 13:04:11 === PODS ===
2026/09/18 13:04:11 Failed to collect pod data: redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-64g2q: container manager not ready
2026/09/18 13:04:11 Continuing with partial pod data to collect logs from problematic containers
2026/09/18 13:04:11 Namespace: redhat-ods-applications
2026/09/18 13:04:11   Pod llmisvc-controller-manager-6cc5fc4bb7-64g2q: Running
2026/09/18 13:04:11     Container manager: ready=false
2026/09/18 13:04:11 Namespace kube-system: no problematic pods found
2026/09/18 13:04:11 === OPERATOR STATUS ===
2026/09/18 13:04:11 Operator deployment: 3/3 ready
2026/09/18 13:04:11 === DSCI STATUS ===
2026/09/18 13:04:11 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; ThanosQuerierAvailable=False; PersesAvailable=False; NodeMetricsEndpointAvailable=False
2026/09/18 13:04:11 === DSC STATUS ===
2026/09/18 13:04:11 Failed to collect DSC data: default-dsc unhealthy conditions: Ready=False; ModulesReady=False; KserveReady=False
2026/09/18 13:04:11 === RECENT EVENTS ===
2026/09/18 13:04:11   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Started - Started container manager
2026/09/18 13:04:11   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 984ms (984ms including waiting). Image size: 238650146 bytes.
2026/09/18 13:04:11   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Created - Created container: manager
2026/09/18 13:04:11   13:04:04 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:04:11   13:04:04 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: AddedInterface - Add eth0 [10.129.45.178/23] from ovn-kubernetes
2026/09/18 13:04:11   13:04:03 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-64g2q to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:04:03 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-64g2q
2026/09/18 13:04:11   13:04:03 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:04:11   13:04:02 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Killing - Stopping container manager
2026/09/18 13:04:11   13:04:02 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Killing - Stopping container manager
2026/09/18 13:04:11   13:03:37 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-kr99f_83f4cae3-4c13-4eb1-a3ba-8afd6fc6cb0f became leader
2026/09/18 13:04:11   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 1.312s (1.312s including waiting). Image size: 240120555 bytes.
2026/09/18 13:04:11   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Created - Created container: manager
2026/09/18 13:04:11   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Started - Started container manager
2026/09/18 13:04:11   13:03:19 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: AddedInterface - Add eth0 [10.129.45.177/23] from ovn-kubernetes
2026/09/18 13:04:11   13:03:19 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:04:11   13:03:18 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-kr99f to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:03:18 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:04:11   13:03:18 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-kr99f
2026/09/18 13:04:11   13:03:17 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Killing - Stopping container manager
2026/09/18 13:04:11   13:03:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Killing - Stopping container manager
2026/09/18 13:04:11   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:11   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container core-bff
2026/09/18 13:04:11   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:11   13:03:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-69fd7c8446 from 1 to 0
2026/09/18 13:04:11   13:03:07 INFO ReplicaSet/rhods-dashboard-69fd7c8446: SuccessfulDelete - Deleted pod: rhods-dashboard-69fd7c8446-jw88c
2026/09/18 13:04:11   13:02:31 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container core-bff
2026/09/18 13:04:11   13:02:31 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: core-bff
2026/09/18 13:04:11   13:02:30 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.081s (1.081s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container kube-rbac-proxy
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.205s (1.205s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container rhods-dashboard
2026/09/18 13:04:11   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: rhods-dashboard
2026/09/18 13:04:11   13:02:28 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:11   13:02:28 INFO Pod/rhods-dashboard-dd84489b7-bxshp: AddedInterface - Add eth0 [10.129.45.176/23] from ovn-kubernetes
2026/09/18 13:04:11   13:02:27 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-bxshp to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container core-bff
2026/09/18 13:04:11   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:11   13:02:27 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 1 to 2
2026/09/18 13:04:11   13:02:27 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulDelete - Deleted pod: rhods-dashboard-68569f858c-kcpdj
2026/09/18 13:04:11   13:02:27 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-68569f858c from 1 to 0
2026/09/18 13:04:11   13:02:27 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-bxshp
2026/09/18 13:04:11   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:11   13:02:23 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Killing - Stopping container kuberay-operator
2026/09/18 13:04:11   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-x5nzf to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Started - Started container kuberay-operator
2026/09/18 13:04:11   13:02:07 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-x5nzf
2026/09/18 13:04:11   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Created - Created container: kuberay-operator
2026/09/18 13:04:11   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:04:11   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: AddedInterface - Add eth0 [10.129.45.175/23] from ovn-kubernetes
2026/09/18 13:04:11   13:02:07 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 0 to 1
2026/09/18 13:04:11   13:02:06 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Killing - Stopping container kuberay-operator
2026/09/18 13:04:11   13:02:06 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Killing - Stopping container kuberay-operator
2026/09/18 13:04:11   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Created - Created container: manager
2026/09/18 13:04:11   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1.054s (1.054s including waiting). Image size: 238841358 bytes.
2026/09/18 13:04:11   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Started - Started container manager
2026/09/18 13:04:11   13:02:01 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:04:11   13:02:01 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: AddedInterface - Add eth0 [10.129.45.174/23] from ovn-kubernetes
2026/09/18 13:04:11   13:02:00 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-7fb7b967bd-2ntmx to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:02:00 INFO ReplicaSet/odh-model-controller-7fb7b967bd: SuccessfulCreate - Created pod: odh-model-controller-7fb7b967bd-2ntmx
2026/09/18 13:04:11   13:02:00 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-7fb7b967bd from 1 to 2
2026/09/18 13:04:11   13:01:51 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: core-bff
2026/09/18 13:04:11   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: AddedInterface - Add eth0 [10.128.15.206/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:3b448f4f3d3ae31edf6bf16b807cfdae344acd64e7790ef6143b918298cd1432" already present on machine
2026/09/18 13:04:11   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Created - Created container: manager
2026/09/18 13:04:11   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Started - Started container manager
2026/09/18 13:04:11   13:01:51 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container core-bff
2026/09/18 13:04:11   13:01:50 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-6f4c9f6fd5-bqf7t to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:50 INFO ReplicaSet/dashboard-operator-6f4c9f6fd5: SuccessfulCreate - Created pod: dashboard-operator-6f4c9f6fd5-bqf7t
2026/09/18 13:04:11   13:01:50 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 2.363s (2.363s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:11   13:01:50 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-6f4c9f6fd5 from 1 to 2
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container kube-rbac-proxy
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container rhods-dashboard
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: rhods-dashboard
2026/09/18 13:04:11   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 927ms (927ms including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:11   13:01:47 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:11   13:01:47 INFO Pod/rhods-dashboard-dd84489b7-cnznb: AddedInterface - Add eth0 [10.128.15.205/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:46 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-cnznb to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container core-bff
2026/09/18 13:04:11   13:01:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 0 to 1
2026/09/18 13:04:11   13:01:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-68569f858c from 2 to 1
2026/09/18 13:04:11   13:01:46 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-cnznb
2026/09/18 13:04:11   13:01:46 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Killing - Stopping container autorag-ui
2026/09/18 13:04:11   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:11   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:11   13:01:46 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulDelete - Deleted pod: rhods-dashboard-68569f858c-9zbrr
2026/09/18 13:04:11   13:01:46 INFO Pod/automl-ui-dfdfcc946-27gns: Killing - Stopping container automl-ui
2026/09/18 13:04:11   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: AddedInterface - Add eth0 [10.128.15.204/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Started - Started container server
2026/09/18 13:04:11   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:04:11   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Created - Created container: server
2026/09/18 13:04:11   13:01:40 INFO Pod/model-serving-api-7b65949446-mjnc4: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-mjnc4 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:40 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 1 to 2
2026/09/18 13:04:11   13:01:40 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-mjnc4
2026/09/18 13:04:11   13:01:37 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:37 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:37 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container core-bff
2026/09/18 13:04:11   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.041s (1.041s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:11   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Started - Started container automl-ui
2026/09/18 13:04:11   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: core-bff
2026/09/18 13:04:11   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Created - Created container: automl-ui
2026/09/18 13:04:11   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f" in 1.451s (1.451s including waiting). Image size: 221650153 bytes.
2026/09/18 13:04:11   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-mlflow-rhel9@sha256:72da9df1353bb583feeb9dee9afa1a4439fba5749bbaf2599c301c0009fc1cb4" in 1.643s (1.643s including waiting). Image size: 179358952 bytes.
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container kube-rbac-proxy
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container rhods-dashboard
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: rhods-dashboard
2026/09/18 13:04:11   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.15s (1.15s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:11   13:01:36 INFO Pod/model-registry-ui-986875f9b-cqjtt: Started - Started container model-registry-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Started - Started container gen-ai-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/model-registry-ui-986875f9b-cqjtt: Created - Created container: model-registry-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Started - Started container mlflow-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Created - Created container: gen-ai-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Created - Created container: mlflow-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:43318525e1012c07e025c225f867db58de51c87eba65d0758fcf06af35cf3799" in 1.572s (1.572s including waiting). Image size: 246518882 bytes.
2026/09/18 13:04:11   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Started - Started container eval-hub-ui
2026/09/18 13:04:11   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-eval-hub-rhel9@sha256:e957c89fc6ff354d10a6b49abd63613ed7a82157cfe0c790b0fa07a883292e6a" in 1.491s (1.491s including waiting). Image size: 187473359 bytes.
2026/09/18 13:04:11   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Created - Created container: eval-hub-ui
2026/09/18 13:04:11   13:01:35 INFO Pod/eval-hub-ui-65bccdf794-m78fg: AddedInterface - Add eth0 [10.128.15.203/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:5e2b0975d8c0be0163e244494f33289fb3e99791798e3b1b1b0fe6421497ee0b" in 1.478s (1.478s including waiting). Image size: 196978981 bytes.
2026/09/18 13:04:11   13:01:35 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-eval-hub-rhel9@sha256:e957c89fc6ff354d10a6b49abd63613ed7a82157cfe0c790b0fa07a883292e6a"
2026/09/18 13:04:11   13:01:35 INFO Pod/model-registry-ui-986875f9b-cqjtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-model-registry-rhel9@sha256:9158b18f63124db7c53589ba1df2b5aee9a32db7465ec785ae37b11af3d049ea" in 1.25s (1.25s including waiting). Image size: 199026570 bytes.
2026/09/18 13:04:11   13:01:35 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: AddedInterface - Add eth0 [10.129.45.173/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:35 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:11   13:01:35 INFO Pod/automl-ui-dfdfcc946-27gns: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f"
2026/09/18 13:04:11   13:01:35 INFO Pod/automl-ui-dfdfcc946-27gns: AddedInterface - Add eth0 [10.129.45.172/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Started - Started container agent-ops-ui
2026/09/18 13:04:11   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Created - Created container: agent-ops-ui
2026/09/18 13:04:11   13:01:34 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-69fd7c8446-jw88c to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:34 INFO Pod/automl-ui-dfdfcc946-27gns: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-dfdfcc946-27gns to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:34 INFO ReplicaSet/rhods-dashboard-69fd7c8446: SuccessfulCreate - Created pod: rhods-dashboard-69fd7c8446-jw88c
2026/09/18 13:04:11   13:01:34 INFO Pod/model-registry-ui-986875f9b-cqjtt: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-model-registry-rhel9@sha256:9158b18f63124db7c53589ba1df2b5aee9a32db7465ec785ae37b11af3d049ea"
2026/09/18 13:04:11   13:01:34 WARN Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "eval-hub-proxy-tls" not found
2026/09/18 13:04:11   13:01:34 INFO Pod/mlflow-ui-688cdfc884-hfxw2: AddedInterface - Add eth0 [10.129.45.171/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:34 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container core-bff
2026/09/18 13:04:11   13:01:34 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: core-bff
2026/09/18 13:04:11   13:01:34 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-mlflow-rhel9@sha256:72da9df1353bb583feeb9dee9afa1a4439fba5749bbaf2599c301c0009fc1cb4"
2026/09/18 13:04:11   13:01:34 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:43318525e1012c07e025c225f867db58de51c87eba65d0758fcf06af35cf3799"
2026/09/18 13:04:11   13:01:34 WARN Pod/automl-ui-dfdfcc946-27gns: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/18 13:04:11   13:01:34 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: AddedInterface - Add eth0 [10.128.15.201/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 949ms (949ms including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:11   13:01:34 INFO ReplicaSet/automl-ui-dfdfcc946: SuccessfulCreate - Created pod: automl-ui-dfdfcc946-27gns
2026/09/18 13:04:11   13:01:34 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-dfdfcc946 from 0 to 1
2026/09/18 13:04:11   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: core-bff
2026/09/18 13:04:11   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container core-bff
2026/09/18 13:04:11   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Started - Started container maas-ui
2026/09/18 13:04:11   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Created - Created container: maas-ui
2026/09/18 13:04:11   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:03a66fd093e7ede531f749d1246d395b52c767bed49c6ff61000a50516137939" in 1.257s (1.257s including waiting). Image size: 189987009 bytes.
2026/09/18 13:04:11   13:01:34 INFO Pod/model-registry-ui-986875f9b-cqjtt: AddedInterface - Add eth0 [10.128.15.202/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-69fd7c8446 from 0 to 1
2026/09/18 13:04:11   13:01:33 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Scheduled - Successfully assigned redhat-ods-applications/eval-hub-ui-65bccdf794-m78fg to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:33 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Scheduled - Successfully assigned redhat-ods-applications/mlflow-ui-688cdfc884-hfxw2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:33 INFO Pod/model-registry-ui-986875f9b-cqjtt: Scheduled - Successfully assigned redhat-ods-applications/model-registry-ui-986875f9b-cqjtt to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Started - Started container autorag-ui
2026/09/18 13:04:11   13:01:33 INFO ReplicaSet/eval-hub-ui-65bccdf794: SuccessfulCreate - Created pod: eval-hub-ui-65bccdf794-m78fg
2026/09/18 13:04:11   13:01:33 INFO Pod/maas-ui-778d4756c9-gfqbk: AddedInterface - Add eth0 [10.129.45.169/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:33 INFO Deployment/model-registry-ui: ScalingReplicaSet - Scaled up replica set model-registry-ui-986875f9b from 0 to 1
2026/09/18 13:04:11   13:01:33 INFO Pod/maas-ui-778d4756c9-gfqbk: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:03a66fd093e7ede531f749d1246d395b52c767bed49c6ff61000a50516137939"
2026/09/18 13:04:11   13:01:33 WARN Pod/model-registry-ui-986875f9b-cqjtt: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "model-registry-proxy-tls" not found
2026/09/18 13:04:11   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container kube-rbac-proxy
2026/09/18 13:04:11   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:11   13:01:33 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:5e2b0975d8c0be0163e244494f33289fb3e99791798e3b1b1b0fe6421497ee0b"
2026/09/18 13:04:11   13:01:33 INFO ReplicaSet/model-registry-ui-986875f9b: SuccessfulCreate - Created pod: model-registry-ui-986875f9b-cqjtt
2026/09/18 13:04:11   13:01:33 INFO Deployment/mlflow-ui: ScalingReplicaSet - Scaled up replica set mlflow-ui-688cdfc884 from 0 to 1
2026/09/18 13:04:11   13:01:33 INFO ReplicaSet/mlflow-ui-688cdfc884: SuccessfulCreate - Created pod: mlflow-ui-688cdfc884-hfxw2
2026/09/18 13:04:11   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Created - Created container: autorag-ui
2026/09/18 13:04:11   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025" in 1.039s (1.039s including waiting). Image size: 203709694 bytes.
2026/09/18 13:04:11   13:01:33 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: AddedInterface - Add eth0 [10.129.45.170/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:33 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.591s (1.591s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:11   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:11   13:01:33 WARN Pod/mlflow-ui-688cdfc884-hfxw2: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "mlflow-proxy-tls" not found
2026/09/18 13:04:11   13:01:33 INFO Deployment/eval-hub-ui: ScalingReplicaSet - Scaled up replica set eval-hub-ui-65bccdf794 from 0 to 1
2026/09/18 13:04:11   13:01:32 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-57b4fbdf48-t6px6 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:32 INFO Pod/maas-ui-778d4756c9-gfqbk: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-778d4756c9-gfqbk to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:32 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-6fdf4d9489-cw8r8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:32 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:11   13:01:32 WARN Pod/gen-ai-ui-6fdf4d9489-cw8r8: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "gen-ai-proxy-tls" not found
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container rhods-dashboard
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: rhods-dashboard
2026/09/18 13:04:11   13:01:32 INFO ReplicaSet/gen-ai-ui-6fdf4d9489: SuccessfulCreate - Created pod: gen-ai-ui-6fdf4d9489-cw8r8
2026/09/18 13:04:11   13:01:32 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-6fdf4d9489 from 0 to 1
2026/09/18 13:04:11   13:01:32 INFO ReplicaSet/agent-ops-ui-57b4fbdf48: SuccessfulCreate - Created pod: agent-ops-ui-57b4fbdf48-t6px6
2026/09/18 13:04:11   13:01:32 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-57b4fbdf48 from 0 to 1
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container kube-rbac-proxy
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.458s (1.458s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:11   13:01:32 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-778d4756c9 from 0 to 1
2026/09/18 13:04:11   13:01:32 INFO ReplicaSet/maas-ui-778d4756c9: SuccessfulCreate - Created pod: maas-ui-778d4756c9-gfqbk
2026/09/18 13:04:11   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Started - Started container kuberay-operator
2026/09/18 13:04:11   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Created - Created container: kuberay-operator
2026/09/18 13:04:11   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:04:11   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: AddedInterface - Add eth0 [10.129.45.168/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container rhods-dashboard
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: rhods-dashboard
2026/09/18 13:04:11   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.014s (1.014s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:11   13:01:32 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: AddedInterface - Add eth0 [10.128.15.200/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:32 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025"
2026/09/18 13:04:11   13:01:31 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-78c4c87bbf-t8j5f to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:31 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:31 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-2c7hq to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:31 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:11   13:01:31 INFO ReplicaSet/autorag-ui-78c4c87bbf: SuccessfulCreate - Created pod: autorag-ui-78c4c87bbf-t8j5f
2026/09/18 13:04:11   13:01:31 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-2c7hq
2026/09/18 13:04:11   13:01:31 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 1 to 2
2026/09/18 13:04:11   13:01:31 INFO Pod/rhods-dashboard-68569f858c-kcpdj: AddedInterface - Add eth0 [10.128.15.199/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:31 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:11   13:01:31 INFO Pod/rhods-dashboard-68569f858c-9zbrr: AddedInterface - Add eth0 [10.129.45.167/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:31 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-78c4c87bbf from 0 to 1
2026/09/18 13:04:11   13:01:31 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-6b5644f98d-9x9df_8b0c9b3f-bc3c-4fe4-8ca3-2ebfe112f55f became leader
2026/09/18 13:04:11   13:01:30 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-68569f858c-9zbrr to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:30 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-68569f858c-kcpdj to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:30 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/18 13:04:11   13:01:30 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulCreate - Created pod: rhods-dashboard-68569f858c-9zbrr
2026/09/18 13:04:11   13:01:30 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulCreate - Created pod: rhods-dashboard-68569f858c-kcpdj
2026/09/18 13:04:11   13:01:30 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-68569f858c from 0 to 2
2026/09/18 13:04:11   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Started - Started container manager
2026/09/18 13:04:11   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Created - Created container: manager
2026/09/18 13:04:11   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 4.976s (4.976s including waiting). Image size: 238650146 bytes.
2026/09/18 13:04:11   13:01:26 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-6f4c9f6fd5-v5llf_d0181e54-6ac3-41a5-beef-01d007de8627 became leader
2026/09/18 13:04:11   13:01:23 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9_8a31aa91-6e9d-4828-9032-14144f389211 became leader
2026/09/18 13:04:11   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Created - Created container: manager
2026/09/18 13:04:11   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: AddedInterface - Add eth0 [10.129.45.166/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:21 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:04:11   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" already present on machine
2026/09/18 13:04:11   13:01:21 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: AddedInterface - Add eth0 [10.128.15.198/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Started - Started container manager
2026/09/18 13:04:11   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:20 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-plxn8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:20 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 1 to 2
2026/09/18 13:04:11   13:01:20 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57lbbcr
2026/09/18 13:04:11   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:20 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 0 to 1
2026/09/18 13:04:11   13:01:20 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-plxn8
2026/09/18 13:04:11   13:01:17 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:17 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:01:16 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Killing - Stopping container training-operator
2026/09/18 13:04:11   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:04:11   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Started - Started container kuberay-operator
2026/09/18 13:04:11   13:01:13 WARN Pod/rhods-dashboard-69fd7c8446-7z84q: Unhealthy - Readiness probe failed: Get "https://10.129.45.22:8943/healthcheck": context deadline exceeded
2026/09/18 13:04:11   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: AddedInterface - Add eth0 [10.128.15.197/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Created - Created container: kuberay-operator
2026/09/18 13:04:11   13:01:12 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-9x9df to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:12 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:12 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 0 to 1
2026/09/18 13:04:11   13:01:12 WARN Pod/kuberay-operator-6b5644f98d-9x9df: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/18 13:04:11   13:01:12 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-9x9df
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Started - Started container nginx
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Created - Created container: nginx
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: AddedInterface - Add eth0 [10.128.15.196/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:a61265350ee8404e4792cd06dd4f1acc7236b5b5c90bff4c543ee6ac02ab4cfd" already present on machine
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Created - Created container: nginx
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Started - Started container nginx
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: AddedInterface - Add eth0 [10.129.45.165/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:a61265350ee8404e4792cd06dd4f1acc7236b5b5c90bff4c543ee6ac02ab4cfd" already present on machine
2026/09/18 13:04:11   13:01:10 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-5c78f45f7-hc4m7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:10 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-5c78f45f7-kvxnc to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Created - Created container: manager
2026/09/18 13:04:11   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: AddedInterface - Add eth0 [10.129.45.164/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:3b448f4f3d3ae31edf6bf16b807cfdae344acd64e7790ef6143b918298cd1432" already present on machine
2026/09/18 13:04:11   13:01:10 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-5c78f45f7 from 0 to 2
2026/09/18 13:04:11   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Started - Started container manager
2026/09/18 13:04:11   13:01:10 INFO ReplicaSet/dashboard-redirect-5c78f45f7: SuccessfulCreate - Created pod: dashboard-redirect-5c78f45f7-hc4m7
2026/09/18 13:04:11   13:01:10 INFO ReplicaSet/dashboard-redirect-5c78f45f7: SuccessfulCreate - Created pod: dashboard-redirect-5c78f45f7-kvxnc
2026/09/18 13:04:11   13:01:09 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-6f4c9f6fd5-v5llf to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:09 INFO ReplicaSet/dashboard-operator-6f4c9f6fd5: SuccessfulCreate - Created pod: dashboard-operator-6f4c9f6fd5-v5llf
2026/09/18 13:04:11   13:01:09 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-6f4c9f6fd5 from 0 to 1
2026/09/18 13:04:11   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: AddedInterface - Add eth0 [10.129.45.163/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Started - Started container manager
2026/09/18 13:04:11   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Created - Created container: manager
2026/09/18 13:04:11   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:04:11   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:05 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 0 to 1
2026/09/18 13:04:11   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:05 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9
2026/09/18 13:04:11   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Started - Started container manager
2026/09/18 13:04:11   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 945ms (945ms including waiting). Image size: 240120555 bytes.
2026/09/18 13:04:11   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Created - Created container: manager
2026/09/18 13:04:11   13:01:01 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:04:11   13:01:01 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: AddedInterface - Add eth0 [10.128.15.195/23] from ovn-kubernetes
2026/09/18 13:04:11   13:01:01 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Killing - Stopping container manager
2026/09/18 13:04:11   13:01:00 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-qqvq8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:01:00 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 1 to 2
2026/09/18 13:04:11   13:01:00 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-qqvq8
2026/09/18 13:04:11   13:00:59 INFO Pod/dashboard-operator-6f4c9f6fd5-bsxjp: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:59 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw_5db8ff17-300e-4d97-a950-cc77ad40f5fd became leader
2026/09/18 13:04:11   13:00:58 WARN Pod/rhods-dashboard-69fd7c8446-7z84q: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = NotFound desc = container is not created or running: checking if PID of cb91b77b69001a0a7ddc3609e04c59c60afda371a164891689713f7b3b6a4abc is running failed: container process not found
2026/09/18 13:04:11   13:00:57 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Killing - Stopping container autorag-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/automl-ui-dfdfcc946-24sv7: Killing - Stopping container automl-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container core-bff
2026/09/18 13:04:11   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:11   13:00:57 INFO Pod/maas-ui-778d4756c9-vpwmd: Killing - Stopping container maas-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/model-registry-ui-986875f9b-h5wj4: Killing - Stopping container model-registry-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/dashboard-redirect-5c78f45f7-hp5th: Killing - Stopping container nginx
2026/09/18 13:04:11   13:00:57 INFO Pod/agent-ops-ui-57b4fbdf48-p9qsd: Killing - Stopping container agent-ops-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container core-bff
2026/09/18 13:04:11   13:00:57 INFO Pod/gen-ai-ui-6fdf4d9489-m7jp5: Killing - Stopping container gen-ai-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/dashboard-redirect-5c78f45f7-zblf4: Killing - Stopping container nginx
2026/09/18 13:04:11   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:11   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:11   13:00:57 INFO Pod/eval-hub-ui-65bccdf794-994bs: Killing - Stopping container eval-hub-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/mlflow-ui-688cdfc884-4f94t: Killing - Stopping container mlflow-ui
2026/09/18 13:04:11   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:11   13:00:54 WARN Pod/kuberay-operator-6b5644f98d-dz8wl: Unhealthy - Readiness probe failed: Get "http://10.128.15.123:8082/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/18 13:04:11   13:00:53 INFO Pod/kuberay-operator-6b5644f98d-dz8wl: Killing - Stopping container kuberay-operator
2026/09/18 13:04:11   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-s7vxx to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: AddedInterface - Add eth0 [10.128.15.194/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Created - Created container: training-operator
2026/09/18 13:04:11   13:00:52 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 0 to 1
2026/09/18 13:04:11   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:04:11   13:00:52 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-s7vxx
2026/09/18 13:04:11   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Started - Started container training-operator
2026/09/18 13:04:11   13:00:50 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:50 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Killing - Stopping container training-operator
2026/09/18 13:04:11   13:00:50 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Killing - Stopping container training-operator
2026/09/18 13:04:11   13:00:49 INFO Pod/workbenches-operator-67897b697-2s98l: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:46 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:46 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:46 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:45 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:45 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Started - Started container manager
2026/09/18 13:04:11   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Created - Created container: manager
2026/09/18 13:04:11   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" in 4.377s (4.377s including waiting). Image size: 178385443 bytes.
2026/09/18 13:04:11   13:00:45 INFO Pod/ai-gateway-operator-57f8cd74c7-f8mps: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Created - Created container: manager
2026/09/18 13:04:11   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Started - Started container manager
2026/09/18 13:04:11   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:c34fec8cac5b49d18d6cd3318d5ebeac91df6e177b0c522a47c1cf0057b008bd" in 1.319s (1.319s including waiting). Image size: 193864580 bytes.
2026/09/18 13:04:11   13:00:41 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: AddedInterface - Add eth0 [10.128.15.193/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:41 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b"
2026/09/18 13:04:11   13:00:40 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:40 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/18 13:04:11   13:00:40 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 1 to 2
2026/09/18 13:04:11   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:c34fec8cac5b49d18d6cd3318d5ebeac91df6e177b0c522a47c1cf0057b008bd"
2026/09/18 13:04:11   13:00:40 INFO ReplicaSet/ogx-k8s-operator-controller-manager-6cbbbfbc9d: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw
2026/09/18 13:04:11   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: AddedInterface - Add eth0 [10.128.15.192/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:40 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-6cbbbfbc9d from 0 to 1
2026/09/18 13:04:11   13:00:40 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57t8kzt
2026/09/18 13:04:11   13:00:39 INFO Pod/maas-controller-f9dfb44dc-245mz: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:37 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-psfpn_b4e0b221-880c-4a02-89f8-334941ced849 became leader
2026/09/18 13:04:11   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f" in 1.285s (1.285s including waiting). Image size: 221650153 bytes.
2026/09/18 13:04:11   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Started - Started container automl-ui
2026/09/18 13:04:11   13:00:36 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-7fb7b967bd-pr77w_35674a84-aec5-4ff3-a77a-fbc41bcfd38c became leader
2026/09/18 13:04:11   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025" in 1.359s (1.359s including waiting). Image size: 203709694 bytes.
2026/09/18 13:04:11   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Created - Created container: autorag-ui
2026/09/18 13:04:11   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Created - Created container: automl-ui
2026/09/18 13:04:11   13:00:36 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6cc5fc4bb7-gfcxq_4c4bce57-75c4-4029-a42e-4b26570680cd became leader
2026/09/18 13:04:11   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Started - Started container autorag-ui
2026/09/18 13:04:11   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container core-bff
2026/09/18 13:04:11   13:00:35 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-dd84489b7 from 1 to 0
2026/09/18 13:04:11   13:00:35 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: AddedInterface - Add eth0 [10.129.45.162/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:35 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulDelete - Deleted pod: rhods-dashboard-dd84489b7-xsh2r
2026/09/18 13:04:11   13:00:35 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025"
2026/09/18 13:04:11   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:11   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:11   13:00:34 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-78c4c87bbf-w9xp6 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-dfdfcc946-24sv7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:34 INFO ReplicaSet/automl-ui-dfdfcc946: SuccessfulCreate - Created pod: automl-ui-dfdfcc946-24sv7
2026/09/18 13:04:11   13:00:34 INFO ReplicaSet/autorag-ui-78c4c87bbf: SuccessfulCreate - Created pod: autorag-ui-78c4c87bbf-w9xp6
2026/09/18 13:04:11   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f"
2026/09/18 13:04:11   13:00:34 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-679df8947-v85j2_2bc2d5b9-e8d2-4ca8-873b-9ac2f17f6fab became leader
2026/09/18 13:04:11   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: AddedInterface - Add eth0 [10.129.45.161/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:34 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-78c4c87bbf from 0 to 1
2026/09/18 13:04:11   13:00:33 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-f9dfb44dc-245mz_325308aa-ba4d-4984-ad9e-980b798a7bb4 became leader
2026/09/18 13:04:11   13:00:33 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-dfdfcc946 from 0 to 1
2026/09/18 13:04:11   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: AddedInterface - Add eth0 [10.128.15.191/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:04:11   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Created - Created container: manager
2026/09/18 13:04:11   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Started - Started container manager
2026/09/18 13:04:11   13:00:29 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:29 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 1 to 2
2026/09/18 13:04:11   13:00:29 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-k48cp
2026/09/18 13:04:11   13:00:27 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.081s (1.081s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:11   13:00:27 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Started - Started container core-bff
2026/09/18 13:04:11   13:00:27 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Created - Created container: core-bff
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Started - Started container rhods-dashboard
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Created - Created container: rhods-dashboard
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.031s (1.031s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Started - Started container kube-rbac-proxy
2026/09/18 13:04:11   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:11   13:00:25 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:11   13:00:25 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: AddedInterface - Add eth0 [10.128.15.190/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:24 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-xsh2r to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:24 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Created - Created container: manager
2026/09/18 13:04:11   13:00:24 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Started - Started container manager
2026/09/18 13:04:11   13:00:24 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-xsh2r
2026/09/18 13:04:11   13:00:24 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Created - Created container: manager
2026/09/18 13:04:11   13:00:24 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 0 to 1
2026/09/18 13:04:11   13:00:24 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:3d7ad3abf865793bb91a53a2543248a03891a9a001d93512d22c59aaf552dace" in 1.14s (1.14s including waiting). Image size: 173556749 bytes.
2026/09/18 13:04:11   13:00:24 INFO Pod/autorag-ui-78c4c87bbf-x6425: Killing - Stopping container autorag-ui
2026/09/18 13:04:11   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Started - Started container manager
2026/09/18 13:04:11   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: AddedInterface - Add eth0 [10.129.45.160/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" already present on machine
2026/09/18 13:04:11   13:00:24 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Started - Started container manager
2026/09/18 13:04:11   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Created - Created container: manager
2026/09/18 13:04:11   13:00:24 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:7fb76908e8ea80e485f52f537beeb7551ce05efe82ed22f5eb1ea1de148389d5" in 1.264s (1.264s including waiting). Image size: 168377417 bytes.
2026/09/18 13:04:11   13:00:23 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57n45qj to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:23 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:23 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:23 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:23 INFO Pod/automl-ui-dfdfcc946-wxwmk: Killing - Stopping container automl-ui
2026/09/18 13:04:11   13:00:23 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57n45qj
2026/09/18 13:04:11   13:00:23 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 0 to 1
2026/09/18 13:04:11   13:00:23 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:3d7ad3abf865793bb91a53a2543248a03891a9a001d93512d22c59aaf552dace"
2026/09/18 13:04:11   13:00:23 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: AddedInterface - Add eth0 [10.129.45.159/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:22 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-87cdb86d9-979zj to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:22 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75574d88c6-v85v8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:22 INFO ReplicaSet/odh-notebook-controller-manager-87cdb86d9: SuccessfulCreate - Created pod: odh-notebook-controller-manager-87cdb86d9-979zj
2026/09/18 13:04:11   13:00:22 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n_ddb2a1d2-3cea-4524-89ad-775ca39dde40 became leader
2026/09/18 13:04:11   13:00:22 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: AddedInterface - Add eth0 [10.128.15.189/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:22 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:7fb76908e8ea80e485f52f537beeb7551ce05efe82ed22f5eb1ea1de148389d5"
2026/09/18 13:04:11   13:00:22 INFO ReplicaSet/notebook-controller-deployment-75574d88c6: SuccessfulCreate - Created pod: notebook-controller-deployment-75574d88c6-v85v8
2026/09/18 13:04:11   13:00:22 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75574d88c6 from 0 to 1
2026/09/18 13:04:11   13:00:22 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Started - Started container manager
2026/09/18 13:04:11   13:00:22 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Created - Created container: manager
2026/09/18 13:04:11   13:00:22 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:90ab74599ab1294fa3e54bd0db517edff2df82b076b103f69b3ef791a75bf9dc" in 5.516s (5.516s including waiting). Image size: 186584994 bytes.
2026/09/18 13:04:11   13:00:22 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-87cdb86d9 from 0 to 1
2026/09/18 13:04:11   13:00:21 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-65d74dc6fc-f894w_669f6d03-4c4b-4d55-89be-4d7963abc46a became leader
2026/09/18 13:04:11   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Created - Created container: training-operator
2026/09/18 13:04:11   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:04:11   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: AddedInterface - Add eth0 [10.128.15.188/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Started - Started container training-operator
2026/09/18 13:04:11   13:00:19 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-4gvd8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:19 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 1 to 2
2026/09/18 13:04:11   13:00:19 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1.035s (1.035s including waiting). Image size: 238841358 bytes.
2026/09/18 13:04:11   13:00:19 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Created - Created container: manager
2026/09/18 13:04:11   13:00:19 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578" in 1.016s (1.016s including waiting). Image size: 194928572 bytes.
2026/09/18 13:04:11   13:00:19 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Started - Started container manager
2026/09/18 13:04:11   13:00:19 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Created - Created container: manager
2026/09/18 13:04:11   13:00:19 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-4gvd8
2026/09/18 13:04:11   13:00:19 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Started - Started container manager
2026/09/18 13:04:11   13:00:19 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Created - Created container: manager
2026/09/18 13:04:11   13:00:19 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Started - Started container manager
2026/09/18 13:04:11   13:00:18 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:18 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:04:11   13:00:18 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: AddedInterface - Add eth0 [10.128.15.187/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:18 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578"
2026/09/18 13:04:11   13:00:18 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 1.044s (1.044s including waiting). Image size: 238650146 bytes.
2026/09/18 13:04:11   13:00:18 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 1.054s (1.054s including waiting). Image size: 240120555 bytes.
2026/09/18 13:04:11   13:00:18 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Started - Started container manager
2026/09/18 13:04:11   13:00:18 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Created - Created container: manager
2026/09/18 13:04:11   13:00:17 INFO Pod/maas-controller-f9dfb44dc-245mz: Created - Created container: manager
2026/09/18 13:04:11   13:00:17 INFO Pod/maas-controller-f9dfb44dc-245mz: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:31870540df9132209a2865ddc3aeff7cc04ea3fdcb044c2e3b3ee18dcb73a688" in 3.304s (3.304s including waiting). Image size: 183271983 bytes.
2026/09/18 13:04:11   13:00:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: AddedInterface - Add eth0 [10.129.45.155/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:17 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Started - Started container copy-manifests
2026/09/18 13:04:11   13:00:17 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Created - Created container: copy-manifests
2026/09/18 13:04:11   13:00:17 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578" in 1.083s (1.083s including waiting). Image size: 194928572 bytes.
2026/09/18 13:04:11   13:00:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:04:11   13:00:17 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: AddedInterface - Add eth0 [10.129.45.156/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:17 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:04:11   13:00:17 INFO Pod/maas-controller-f9dfb44dc-245mz: Started - Started container manager
2026/09/18 13:04:11   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: AddedInterface - Add eth0 [10.129.45.158/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:04:11   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: Created - Created container: server
2026/09/18 13:04:11   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: Started - Started container server
2026/09/18 13:04:11   13:00:16 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-7fb7b967bd-pr77w to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:16 INFO Pod/model-serving-api-7b65949446-lhg7k: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-lhg7k to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:16 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-gfcxq to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:16 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:16 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-psfpn to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:16 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: AddedInterface - Add eth0 [10.129.45.153/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:16 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578"
2026/09/18 13:04:11   13:00:16 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-lhg7k
2026/09/18 13:04:11   13:00:16 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-7fb7b967bd from 0 to 1
2026/09/18 13:04:11   13:00:16 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: AddedInterface - Add eth0 [10.129.45.154/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:16 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 0 to 1
2026/09/18 13:04:11   13:00:16 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:04:11   13:00:16 WARN Pod/model-serving-api-7b65949446-lhg7k: ErrorUpdatingResource - addLogicalPort failed for redhat-ods-applications/model-serving-api-7b65949446-lhg7k: failed to update pod redhat-ods-applications/model-serving-api-7b65949446-lhg7k: the server rejected our request due to an error in our request
2026/09/18 13:04:11   13:00:16 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-psfpn
2026/09/18 13:04:11   13:00:16 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:04:11   13:00:16 INFO ReplicaSet/odh-model-controller-7fb7b967bd: SuccessfulCreate - Created pod: odh-model-controller-7fb7b967bd-pr77w
2026/09/18 13:04:11   13:00:16 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:90ab74599ab1294fa3e54bd0db517edff2df82b076b103f69b3ef791a75bf9dc"
2026/09/18 13:04:11   13:00:16 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-gfcxq
2026/09/18 13:04:11   13:00:15 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:15 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:15 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:15 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-679df8947-v85j2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:15 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-7cfbff9c75: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n
2026/09/18 13:04:11   13:00:15 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-679df8947 from 0 to 1
2026/09/18 13:04:11   13:00:15 INFO ReplicaSet/opendatahub-ogx-operator-679df8947: SuccessfulCreate - Created pod: opendatahub-ogx-operator-679df8947-v85j2
2026/09/18 13:04:11   13:00:15 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-7cfbff9c75 from 0 to 1
2026/09/18 13:04:11   13:00:14 INFO Pod/workbenches-operator-67897b697-2s98l: Created - Created container: manager
2026/09/18 13:04:11   13:00:14 INFO Pod/workbenches-operator-67897b697-2s98l: Started - Started container manager
2026/09/18 13:04:11   13:00:14 INFO Pod/maas-controller-f9dfb44dc-245mz: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:31870540df9132209a2865ddc3aeff7cc04ea3fdcb044c2e3b3ee18dcb73a688"
2026/09/18 13:04:11   13:00:14 INFO Pod/maas-controller-f9dfb44dc-245mz: AddedInterface - Add eth0 [10.129.45.152/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:13 INFO Pod/maas-controller-f9dfb44dc-245mz: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-f9dfb44dc-245mz to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:13 INFO Pod/workbenches-operator-67897b697-2s98l: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:96c73966142a6cbf5a8b3dd69cbf92cae432987edde6a1cc39aa200915e7772c" already present on machine
2026/09/18 13:04:11   13:00:13 INFO ReplicaSet/maas-controller-f9dfb44dc: SuccessfulCreate - Created pod: maas-controller-f9dfb44dc-245mz
2026/09/18 13:04:11   13:00:13 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-f9dfb44dc from 0 to 1
2026/09/18 13:04:11   13:00:12 INFO Pod/workbenches-operator-67897b697-2s98l: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-67897b697-2s98l to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:12 INFO Pod/workbenches-operator-67897b697-2s98l: AddedInterface - Add eth0 [10.129.45.151/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:12 INFO ReplicaSet/workbenches-operator-67897b697: SuccessfulCreate - Created pod: workbenches-operator-67897b697-2s98l
2026/09/18 13:04:11   13:00:12 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-67897b697 from 0 to 1
2026/09/18 13:04:11   13:00:11 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Created - Created container: manager
2026/09/18 13:04:11   13:00:11 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Started - Started container manager
2026/09/18 13:04:11   13:00:11 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:fd4eae35eb2e80004d87d692bdf27f5bab20b0673011006307c0d54210934c84" in 4.497s (4.497s including waiting). Image size: 143072345 bytes.
2026/09/18 13:04:11   13:00:11 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8_966488f1-a994-4fa7-9de2-49eb9746e7bb became leader
2026/09/18 13:04:11   13:00:10 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:10 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57hbm9s: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:08 INFO Pod/opendatahub-feast-operator-5f45cd9467-nq89j: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:07 INFO Pod/opendatahub-ogx-operator-679df8947-4r8z2: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:06 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:06 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-bdc596ccc: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8
2026/09/18 13:04:11   13:00:06 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:fd4eae35eb2e80004d87d692bdf27f5bab20b0673011006307c0d54210934c84"
2026/09/18 13:04:11   13:00:06 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: AddedInterface - Add eth0 [10.129.45.150/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:06 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-bdc596ccc from 0 to 1
2026/09/18 13:04:11   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: AddedInterface - Add eth0 [10.129.45.149/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Started - Started container manager
2026/09/18 13:04:11   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Created - Created container: manager
2026/09/18 13:04:11   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:04:11   13:00:04 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:04 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-f894w to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   13:00:04 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 0 to 1
2026/09/18 13:04:11   13:00:04 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-f894w
2026/09/18 13:04:11   13:00:03 INFO Pod/kserve-controller-manager-75b9f799cd-j7q2h: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:03 INFO Pod/feast-operator-controller-manager-8fbf6785c-rpxbg: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:03 INFO Pod/odh-model-controller-7fb7b967bd-mj2g2: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:03 INFO Pod/model-serving-api-7b65949446-65hzr: Killing - Stopping container server
2026/09/18 13:04:11   13:00:03 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-5xd7k: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:02 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:02 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   13:00:00 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-g6chf: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:00 WARN Pod/workbenches-operator-67897b697-6dk8j: Unhealthy - Readiness probe failed: Get "http://10.128.15.121:8081/readyz": dial tcp 10.128.15.121:8081: connect: connection refused
2026/09/18 13:04:11   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: AddedInterface - Add eth0 [10.129.45.147/23] from ovn-kubernetes
2026/09/18 13:04:11   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:04:11   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Created - Created container: training-operator
2026/09/18 13:04:11   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Started - Started container training-operator
2026/09/18 13:04:11   13:00:00 INFO Pod/workbenches-operator-67897b697-6dk8j: Killing - Stopping container manager
2026/09/18 13:04:11   13:00:00 INFO Pod/maas-controller-f9dfb44dc-pc2jm: Killing - Stopping container manager
2026/09/18 13:04:11   12:59:59 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-dclv2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:11   12:59:59 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-dclv2
2026/09/18 13:04:11   12:59:58 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   12:59:58 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   12:59:58 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 0 to 1
2026/09/18 13:04:11   12:59:53 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-2r8qh: Killing - Stopping container manager
2026/09/18 13:04:11   12:59:52 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   12:59:52 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:11   12:59:49 INFO Pod/notebook-controller-deployment-75574d88c6-zkdn5: Killing - Stopping container manager
2026/09/18 13:04:11   12:59:49 INFO Pod/odh-notebook-controller-manager-87cdb86d9-s99zz: Killing - Stopping container manager
2026/09/18 13:04:11 === RESOURCE QUOTAS ===
2026/09/18 13:04:11 No resource quotas found in monitored namespaces
2026/09/18 13:04:11 Classification for ConfigMap deletion recovery: infrastructure/storage (code=1006, confidence=medium)
2026/09/18 13:04:11   Evidence: event Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "eval-hub-proxy-tls" not found
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"storage","error_code":1006,"evidence":["event Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume \"proxy-tls\" : secret \"eval-hub-proxy-tls\" not found"],"confidence":"medium"}
2026/09/18 13:04:11 Diagnostics complete for failed test: ConfigMap deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/18 13:04:11 TEST FAILURE DETECTED: Validate resource deletion recovery
2026/09/18 13:04:11 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:Validate resource deletion recovery) ===
2026/09/18 13:04:12 === CLUSTER STATE ===
2026/09/18 13:04:12 Node: ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:04:12   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:04:12   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:04:12   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:04:12   Resources: 15500m CPU, 63871024Ki memory allocatable (of 16000m CPU, 65022000Ki memory capacity)
2026/09/18 13:04:12 Node: ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:04:12   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:04:12   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:04:12   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:04:12   Resources: 15500m CPU, 63182884Ki memory allocatable (of 16000m CPU, 64333860Ki memory capacity)
2026/09/18 13:04:12 === DEPLOYMENTS ===
2026/09/18 13:04:12 Failed to collect deployment data: deployments not ready: redhat-ods-applications/llmisvc-controller-manager (0/1)
2026/09/18 13:04:12 === PODS ===
2026/09/18 13:04:12 Failed to collect pod data: redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-64g2q: container manager not ready
2026/09/18 13:04:12 Continuing with partial pod data to collect logs from problematic containers
2026/09/18 13:04:12 Namespace: redhat-ods-applications
2026/09/18 13:04:12   Pod llmisvc-controller-manager-6cc5fc4bb7-64g2q: Running
2026/09/18 13:04:12     Container manager: ready=false
2026/09/18 13:04:12 Namespace kube-system: no problematic pods found
2026/09/18 13:04:12 === OPERATOR STATUS ===
2026/09/18 13:04:12 Operator deployment: 3/3 ready
2026/09/18 13:04:12 === DSCI STATUS ===
2026/09/18 13:04:12 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; ThanosQuerierAvailable=False; PersesAvailable=False; NodeMetricsEndpointAvailable=False
2026/09/18 13:04:12 === DSC STATUS ===
2026/09/18 13:04:12 Failed to collect DSC data: default-dsc unhealthy conditions: Ready=False; ModulesReady=False; KserveReady=False
2026/09/18 13:04:12 === RECENT EVENTS ===
2026/09/18 13:04:12   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Started - Started container manager
2026/09/18 13:04:12   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 984ms (984ms including waiting). Image size: 238650146 bytes.
2026/09/18 13:04:12   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Created - Created container: manager
2026/09/18 13:04:12   13:04:04 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:04:12   13:04:04 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: AddedInterface - Add eth0 [10.129.45.178/23] from ovn-kubernetes
2026/09/18 13:04:12   13:04:03 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-64g2q to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:04:03 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-64g2q
2026/09/18 13:04:12   13:04:03 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:04:12   13:04:02 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Killing - Stopping container manager
2026/09/18 13:04:12   13:04:02 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Killing - Stopping container manager
2026/09/18 13:04:12   13:03:37 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-kr99f_83f4cae3-4c13-4eb1-a3ba-8afd6fc6cb0f became leader
2026/09/18 13:04:12   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 1.312s (1.312s including waiting). Image size: 240120555 bytes.
2026/09/18 13:04:12   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Created - Created container: manager
2026/09/18 13:04:12   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Started - Started container manager
2026/09/18 13:04:12   13:03:19 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: AddedInterface - Add eth0 [10.129.45.177/23] from ovn-kubernetes
2026/09/18 13:04:12   13:03:19 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:04:12   13:03:18 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-kr99f to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:03:18 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:04:12   13:03:18 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-kr99f
2026/09/18 13:04:12   13:03:17 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Killing - Stopping container manager
2026/09/18 13:04:12   13:03:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Killing - Stopping container manager
2026/09/18 13:04:12   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:12   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container core-bff
2026/09/18 13:04:12   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:12   13:03:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-69fd7c8446 from 1 to 0
2026/09/18 13:04:12   13:03:07 INFO ReplicaSet/rhods-dashboard-69fd7c8446: SuccessfulDelete - Deleted pod: rhods-dashboard-69fd7c8446-jw88c
2026/09/18 13:04:12   13:02:31 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container core-bff
2026/09/18 13:04:12   13:02:31 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: core-bff
2026/09/18 13:04:12   13:02:30 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.081s (1.081s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container kube-rbac-proxy
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.205s (1.205s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container rhods-dashboard
2026/09/18 13:04:12   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: rhods-dashboard
2026/09/18 13:04:12   13:02:28 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:12   13:02:28 INFO Pod/rhods-dashboard-dd84489b7-bxshp: AddedInterface - Add eth0 [10.129.45.176/23] from ovn-kubernetes
2026/09/18 13:04:12   13:02:27 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-bxshp to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container core-bff
2026/09/18 13:04:12   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:12   13:02:27 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 1 to 2
2026/09/18 13:04:12   13:02:27 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulDelete - Deleted pod: rhods-dashboard-68569f858c-kcpdj
2026/09/18 13:04:12   13:02:27 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-68569f858c from 1 to 0
2026/09/18 13:04:12   13:02:27 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-bxshp
2026/09/18 13:04:12   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:12   13:02:23 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Killing - Stopping container kuberay-operator
2026/09/18 13:04:12   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-x5nzf to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Started - Started container kuberay-operator
2026/09/18 13:04:12   13:02:07 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-x5nzf
2026/09/18 13:04:12   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Created - Created container: kuberay-operator
2026/09/18 13:04:12   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:04:12   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: AddedInterface - Add eth0 [10.129.45.175/23] from ovn-kubernetes
2026/09/18 13:04:12   13:02:07 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 0 to 1
2026/09/18 13:04:12   13:02:06 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Killing - Stopping container kuberay-operator
2026/09/18 13:04:12   13:02:06 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Killing - Stopping container kuberay-operator
2026/09/18 13:04:12   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Created - Created container: manager
2026/09/18 13:04:12   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1.054s (1.054s including waiting). Image size: 238841358 bytes.
2026/09/18 13:04:12   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Started - Started container manager
2026/09/18 13:04:12   13:02:01 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:04:12   13:02:01 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: AddedInterface - Add eth0 [10.129.45.174/23] from ovn-kubernetes
2026/09/18 13:04:12   13:02:00 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-7fb7b967bd-2ntmx to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:02:00 INFO ReplicaSet/odh-model-controller-7fb7b967bd: SuccessfulCreate - Created pod: odh-model-controller-7fb7b967bd-2ntmx
2026/09/18 13:04:12   13:02:00 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-7fb7b967bd from 1 to 2
2026/09/18 13:04:12   13:01:51 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: core-bff
2026/09/18 13:04:12   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: AddedInterface - Add eth0 [10.128.15.206/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:3b448f4f3d3ae31edf6bf16b807cfdae344acd64e7790ef6143b918298cd1432" already present on machine
2026/09/18 13:04:12   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Created - Created container: manager
2026/09/18 13:04:12   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Started - Started container manager
2026/09/18 13:04:12   13:01:51 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container core-bff
2026/09/18 13:04:12   13:01:50 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-6f4c9f6fd5-bqf7t to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:50 INFO ReplicaSet/dashboard-operator-6f4c9f6fd5: SuccessfulCreate - Created pod: dashboard-operator-6f4c9f6fd5-bqf7t
2026/09/18 13:04:12   13:01:50 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 2.363s (2.363s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:12   13:01:50 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-6f4c9f6fd5 from 1 to 2
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container kube-rbac-proxy
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container rhods-dashboard
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: rhods-dashboard
2026/09/18 13:04:12   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 927ms (927ms including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:12   13:01:47 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:12   13:01:47 INFO Pod/rhods-dashboard-dd84489b7-cnznb: AddedInterface - Add eth0 [10.128.15.205/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:46 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-cnznb to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container core-bff
2026/09/18 13:04:12   13:01:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 0 to 1
2026/09/18 13:04:12   13:01:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-68569f858c from 2 to 1
2026/09/18 13:04:12   13:01:46 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-cnznb
2026/09/18 13:04:12   13:01:46 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Killing - Stopping container autorag-ui
2026/09/18 13:04:12   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:12   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:12   13:01:46 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulDelete - Deleted pod: rhods-dashboard-68569f858c-9zbrr
2026/09/18 13:04:12   13:01:46 INFO Pod/automl-ui-dfdfcc946-27gns: Killing - Stopping container automl-ui
2026/09/18 13:04:12   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: AddedInterface - Add eth0 [10.128.15.204/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Started - Started container server
2026/09/18 13:04:12   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:04:12   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Created - Created container: server
2026/09/18 13:04:12   13:01:40 INFO Pod/model-serving-api-7b65949446-mjnc4: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-mjnc4 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:40 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 1 to 2
2026/09/18 13:04:12   13:01:40 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-mjnc4
2026/09/18 13:04:12   13:01:37 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:37 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:37 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container core-bff
2026/09/18 13:04:12   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.041s (1.041s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:12   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Started - Started container automl-ui
2026/09/18 13:04:12   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: core-bff
2026/09/18 13:04:12   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Created - Created container: automl-ui
2026/09/18 13:04:12   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f" in 1.451s (1.451s including waiting). Image size: 221650153 bytes.
2026/09/18 13:04:12   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-mlflow-rhel9@sha256:72da9df1353bb583feeb9dee9afa1a4439fba5749bbaf2599c301c0009fc1cb4" in 1.643s (1.643s including waiting). Image size: 179358952 bytes.
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container kube-rbac-proxy
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container rhods-dashboard
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: rhods-dashboard
2026/09/18 13:04:12   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.15s (1.15s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:12   13:01:36 INFO Pod/model-registry-ui-986875f9b-cqjtt: Started - Started container model-registry-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Started - Started container gen-ai-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/model-registry-ui-986875f9b-cqjtt: Created - Created container: model-registry-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Started - Started container mlflow-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Created - Created container: gen-ai-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Created - Created container: mlflow-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:43318525e1012c07e025c225f867db58de51c87eba65d0758fcf06af35cf3799" in 1.572s (1.572s including waiting). Image size: 246518882 bytes.
2026/09/18 13:04:12   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Started - Started container eval-hub-ui
2026/09/18 13:04:12   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-eval-hub-rhel9@sha256:e957c89fc6ff354d10a6b49abd63613ed7a82157cfe0c790b0fa07a883292e6a" in 1.491s (1.491s including waiting). Image size: 187473359 bytes.
2026/09/18 13:04:12   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Created - Created container: eval-hub-ui
2026/09/18 13:04:12   13:01:35 INFO Pod/eval-hub-ui-65bccdf794-m78fg: AddedInterface - Add eth0 [10.128.15.203/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:5e2b0975d8c0be0163e244494f33289fb3e99791798e3b1b1b0fe6421497ee0b" in 1.478s (1.478s including waiting). Image size: 196978981 bytes.
2026/09/18 13:04:12   13:01:35 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-eval-hub-rhel9@sha256:e957c89fc6ff354d10a6b49abd63613ed7a82157cfe0c790b0fa07a883292e6a"
2026/09/18 13:04:12   13:01:35 INFO Pod/model-registry-ui-986875f9b-cqjtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-model-registry-rhel9@sha256:9158b18f63124db7c53589ba1df2b5aee9a32db7465ec785ae37b11af3d049ea" in 1.25s (1.25s including waiting). Image size: 199026570 bytes.
2026/09/18 13:04:12   13:01:35 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: AddedInterface - Add eth0 [10.129.45.173/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:35 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:12   13:01:35 INFO Pod/automl-ui-dfdfcc946-27gns: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f"
2026/09/18 13:04:12   13:01:35 INFO Pod/automl-ui-dfdfcc946-27gns: AddedInterface - Add eth0 [10.129.45.172/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Started - Started container agent-ops-ui
2026/09/18 13:04:12   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Created - Created container: agent-ops-ui
2026/09/18 13:04:12   13:01:34 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-69fd7c8446-jw88c to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:34 INFO Pod/automl-ui-dfdfcc946-27gns: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-dfdfcc946-27gns to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:34 INFO ReplicaSet/rhods-dashboard-69fd7c8446: SuccessfulCreate - Created pod: rhods-dashboard-69fd7c8446-jw88c
2026/09/18 13:04:12   13:01:34 INFO Pod/model-registry-ui-986875f9b-cqjtt: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-model-registry-rhel9@sha256:9158b18f63124db7c53589ba1df2b5aee9a32db7465ec785ae37b11af3d049ea"
2026/09/18 13:04:12   13:01:34 WARN Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "eval-hub-proxy-tls" not found
2026/09/18 13:04:12   13:01:34 INFO Pod/mlflow-ui-688cdfc884-hfxw2: AddedInterface - Add eth0 [10.129.45.171/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:34 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container core-bff
2026/09/18 13:04:12   13:01:34 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: core-bff
2026/09/18 13:04:12   13:01:34 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-mlflow-rhel9@sha256:72da9df1353bb583feeb9dee9afa1a4439fba5749bbaf2599c301c0009fc1cb4"
2026/09/18 13:04:12   13:01:34 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:43318525e1012c07e025c225f867db58de51c87eba65d0758fcf06af35cf3799"
2026/09/18 13:04:12   13:01:34 WARN Pod/automl-ui-dfdfcc946-27gns: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/18 13:04:12   13:01:34 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: AddedInterface - Add eth0 [10.128.15.201/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 949ms (949ms including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:12   13:01:34 INFO ReplicaSet/automl-ui-dfdfcc946: SuccessfulCreate - Created pod: automl-ui-dfdfcc946-27gns
2026/09/18 13:04:12   13:01:34 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-dfdfcc946 from 0 to 1
2026/09/18 13:04:12   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: core-bff
2026/09/18 13:04:12   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container core-bff
2026/09/18 13:04:12   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Started - Started container maas-ui
2026/09/18 13:04:12   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Created - Created container: maas-ui
2026/09/18 13:04:12   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:03a66fd093e7ede531f749d1246d395b52c767bed49c6ff61000a50516137939" in 1.257s (1.257s including waiting). Image size: 189987009 bytes.
2026/09/18 13:04:12   13:01:34 INFO Pod/model-registry-ui-986875f9b-cqjtt: AddedInterface - Add eth0 [10.128.15.202/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-69fd7c8446 from 0 to 1
2026/09/18 13:04:12   13:01:33 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Scheduled - Successfully assigned redhat-ods-applications/eval-hub-ui-65bccdf794-m78fg to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:33 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Scheduled - Successfully assigned redhat-ods-applications/mlflow-ui-688cdfc884-hfxw2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:33 INFO Pod/model-registry-ui-986875f9b-cqjtt: Scheduled - Successfully assigned redhat-ods-applications/model-registry-ui-986875f9b-cqjtt to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Started - Started container autorag-ui
2026/09/18 13:04:12   13:01:33 INFO ReplicaSet/eval-hub-ui-65bccdf794: SuccessfulCreate - Created pod: eval-hub-ui-65bccdf794-m78fg
2026/09/18 13:04:12   13:01:33 INFO Pod/maas-ui-778d4756c9-gfqbk: AddedInterface - Add eth0 [10.129.45.169/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:33 INFO Deployment/model-registry-ui: ScalingReplicaSet - Scaled up replica set model-registry-ui-986875f9b from 0 to 1
2026/09/18 13:04:12   13:01:33 INFO Pod/maas-ui-778d4756c9-gfqbk: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:03a66fd093e7ede531f749d1246d395b52c767bed49c6ff61000a50516137939"
2026/09/18 13:04:12   13:01:33 WARN Pod/model-registry-ui-986875f9b-cqjtt: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "model-registry-proxy-tls" not found
2026/09/18 13:04:12   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container kube-rbac-proxy
2026/09/18 13:04:12   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:12   13:01:33 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:5e2b0975d8c0be0163e244494f33289fb3e99791798e3b1b1b0fe6421497ee0b"
2026/09/18 13:04:12   13:01:33 INFO ReplicaSet/model-registry-ui-986875f9b: SuccessfulCreate - Created pod: model-registry-ui-986875f9b-cqjtt
2026/09/18 13:04:12   13:01:33 INFO Deployment/mlflow-ui: ScalingReplicaSet - Scaled up replica set mlflow-ui-688cdfc884 from 0 to 1
2026/09/18 13:04:12   13:01:33 INFO ReplicaSet/mlflow-ui-688cdfc884: SuccessfulCreate - Created pod: mlflow-ui-688cdfc884-hfxw2
2026/09/18 13:04:12   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Created - Created container: autorag-ui
2026/09/18 13:04:12   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025" in 1.039s (1.039s including waiting). Image size: 203709694 bytes.
2026/09/18 13:04:12   13:01:33 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: AddedInterface - Add eth0 [10.129.45.170/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:33 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.591s (1.591s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:12   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:12   13:01:33 WARN Pod/mlflow-ui-688cdfc884-hfxw2: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "mlflow-proxy-tls" not found
2026/09/18 13:04:12   13:01:33 INFO Deployment/eval-hub-ui: ScalingReplicaSet - Scaled up replica set eval-hub-ui-65bccdf794 from 0 to 1
2026/09/18 13:04:12   13:01:32 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-57b4fbdf48-t6px6 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:32 INFO Pod/maas-ui-778d4756c9-gfqbk: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-778d4756c9-gfqbk to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:32 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-6fdf4d9489-cw8r8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:32 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:12   13:01:32 WARN Pod/gen-ai-ui-6fdf4d9489-cw8r8: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "gen-ai-proxy-tls" not found
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container rhods-dashboard
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: rhods-dashboard
2026/09/18 13:04:12   13:01:32 INFO ReplicaSet/gen-ai-ui-6fdf4d9489: SuccessfulCreate - Created pod: gen-ai-ui-6fdf4d9489-cw8r8
2026/09/18 13:04:12   13:01:32 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-6fdf4d9489 from 0 to 1
2026/09/18 13:04:12   13:01:32 INFO ReplicaSet/agent-ops-ui-57b4fbdf48: SuccessfulCreate - Created pod: agent-ops-ui-57b4fbdf48-t6px6
2026/09/18 13:04:12   13:01:32 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-57b4fbdf48 from 0 to 1
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container kube-rbac-proxy
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.458s (1.458s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:12   13:01:32 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-778d4756c9 from 0 to 1
2026/09/18 13:04:12   13:01:32 INFO ReplicaSet/maas-ui-778d4756c9: SuccessfulCreate - Created pod: maas-ui-778d4756c9-gfqbk
2026/09/18 13:04:12   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Started - Started container kuberay-operator
2026/09/18 13:04:12   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Created - Created container: kuberay-operator
2026/09/18 13:04:12   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:04:12   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: AddedInterface - Add eth0 [10.129.45.168/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container rhods-dashboard
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: rhods-dashboard
2026/09/18 13:04:12   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.014s (1.014s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:12   13:01:32 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: AddedInterface - Add eth0 [10.128.15.200/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:32 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025"
2026/09/18 13:04:12   13:01:31 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-78c4c87bbf-t8j5f to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:31 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:31 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-2c7hq to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:31 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:12   13:01:31 INFO ReplicaSet/autorag-ui-78c4c87bbf: SuccessfulCreate - Created pod: autorag-ui-78c4c87bbf-t8j5f
2026/09/18 13:04:12   13:01:31 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-2c7hq
2026/09/18 13:04:12   13:01:31 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 1 to 2
2026/09/18 13:04:12   13:01:31 INFO Pod/rhods-dashboard-68569f858c-kcpdj: AddedInterface - Add eth0 [10.128.15.199/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:31 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:12   13:01:31 INFO Pod/rhods-dashboard-68569f858c-9zbrr: AddedInterface - Add eth0 [10.129.45.167/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:31 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-78c4c87bbf from 0 to 1
2026/09/18 13:04:12   13:01:31 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-6b5644f98d-9x9df_8b0c9b3f-bc3c-4fe4-8ca3-2ebfe112f55f became leader
2026/09/18 13:04:12   13:01:30 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-68569f858c-9zbrr to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:30 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-68569f858c-kcpdj to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:30 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/18 13:04:12   13:01:30 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulCreate - Created pod: rhods-dashboard-68569f858c-9zbrr
2026/09/18 13:04:12   13:01:30 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulCreate - Created pod: rhods-dashboard-68569f858c-kcpdj
2026/09/18 13:04:12   13:01:30 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-68569f858c from 0 to 2
2026/09/18 13:04:12   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Started - Started container manager
2026/09/18 13:04:12   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Created - Created container: manager
2026/09/18 13:04:12   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 4.976s (4.976s including waiting). Image size: 238650146 bytes.
2026/09/18 13:04:12   13:01:26 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-6f4c9f6fd5-v5llf_d0181e54-6ac3-41a5-beef-01d007de8627 became leader
2026/09/18 13:04:12   13:01:23 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9_8a31aa91-6e9d-4828-9032-14144f389211 became leader
2026/09/18 13:04:12   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Created - Created container: manager
2026/09/18 13:04:12   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: AddedInterface - Add eth0 [10.129.45.166/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:21 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:04:12   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" already present on machine
2026/09/18 13:04:12   13:01:21 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: AddedInterface - Add eth0 [10.128.15.198/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Started - Started container manager
2026/09/18 13:04:12   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:20 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-plxn8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:20 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 1 to 2
2026/09/18 13:04:12   13:01:20 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57lbbcr
2026/09/18 13:04:12   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:20 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 0 to 1
2026/09/18 13:04:12   13:01:20 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-plxn8
2026/09/18 13:04:12   13:01:17 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:17 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:01:16 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Killing - Stopping container training-operator
2026/09/18 13:04:12   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:04:12   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Started - Started container kuberay-operator
2026/09/18 13:04:12   13:01:13 WARN Pod/rhods-dashboard-69fd7c8446-7z84q: Unhealthy - Readiness probe failed: Get "https://10.129.45.22:8943/healthcheck": context deadline exceeded
2026/09/18 13:04:12   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: AddedInterface - Add eth0 [10.128.15.197/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Created - Created container: kuberay-operator
2026/09/18 13:04:12   13:01:12 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-9x9df to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:12 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:12 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 0 to 1
2026/09/18 13:04:12   13:01:12 WARN Pod/kuberay-operator-6b5644f98d-9x9df: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/18 13:04:12   13:01:12 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-9x9df
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Started - Started container nginx
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Created - Created container: nginx
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: AddedInterface - Add eth0 [10.128.15.196/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:a61265350ee8404e4792cd06dd4f1acc7236b5b5c90bff4c543ee6ac02ab4cfd" already present on machine
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Created - Created container: nginx
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Started - Started container nginx
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: AddedInterface - Add eth0 [10.129.45.165/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:a61265350ee8404e4792cd06dd4f1acc7236b5b5c90bff4c543ee6ac02ab4cfd" already present on machine
2026/09/18 13:04:12   13:01:10 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-5c78f45f7-hc4m7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:10 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-5c78f45f7-kvxnc to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Created - Created container: manager
2026/09/18 13:04:12   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: AddedInterface - Add eth0 [10.129.45.164/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:3b448f4f3d3ae31edf6bf16b807cfdae344acd64e7790ef6143b918298cd1432" already present on machine
2026/09/18 13:04:12   13:01:10 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-5c78f45f7 from 0 to 2
2026/09/18 13:04:12   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Started - Started container manager
2026/09/18 13:04:12   13:01:10 INFO ReplicaSet/dashboard-redirect-5c78f45f7: SuccessfulCreate - Created pod: dashboard-redirect-5c78f45f7-hc4m7
2026/09/18 13:04:12   13:01:10 INFO ReplicaSet/dashboard-redirect-5c78f45f7: SuccessfulCreate - Created pod: dashboard-redirect-5c78f45f7-kvxnc
2026/09/18 13:04:12   13:01:09 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-6f4c9f6fd5-v5llf to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:09 INFO ReplicaSet/dashboard-operator-6f4c9f6fd5: SuccessfulCreate - Created pod: dashboard-operator-6f4c9f6fd5-v5llf
2026/09/18 13:04:12   13:01:09 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-6f4c9f6fd5 from 0 to 1
2026/09/18 13:04:12   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: AddedInterface - Add eth0 [10.129.45.163/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Started - Started container manager
2026/09/18 13:04:12   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Created - Created container: manager
2026/09/18 13:04:12   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:04:12   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:05 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 0 to 1
2026/09/18 13:04:12   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:05 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9
2026/09/18 13:04:12   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Started - Started container manager
2026/09/18 13:04:12   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 945ms (945ms including waiting). Image size: 240120555 bytes.
2026/09/18 13:04:12   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Created - Created container: manager
2026/09/18 13:04:12   13:01:01 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:04:12   13:01:01 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: AddedInterface - Add eth0 [10.128.15.195/23] from ovn-kubernetes
2026/09/18 13:04:12   13:01:01 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Killing - Stopping container manager
2026/09/18 13:04:12   13:01:00 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-qqvq8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:01:00 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 1 to 2
2026/09/18 13:04:12   13:01:00 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-qqvq8
2026/09/18 13:04:12   13:00:59 INFO Pod/dashboard-operator-6f4c9f6fd5-bsxjp: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:59 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw_5db8ff17-300e-4d97-a950-cc77ad40f5fd became leader
2026/09/18 13:04:12   13:00:58 WARN Pod/rhods-dashboard-69fd7c8446-7z84q: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = NotFound desc = container is not created or running: checking if PID of cb91b77b69001a0a7ddc3609e04c59c60afda371a164891689713f7b3b6a4abc is running failed: container process not found
2026/09/18 13:04:12   13:00:57 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Killing - Stopping container autorag-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/automl-ui-dfdfcc946-24sv7: Killing - Stopping container automl-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container core-bff
2026/09/18 13:04:12   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:12   13:00:57 INFO Pod/maas-ui-778d4756c9-vpwmd: Killing - Stopping container maas-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/model-registry-ui-986875f9b-h5wj4: Killing - Stopping container model-registry-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/dashboard-redirect-5c78f45f7-hp5th: Killing - Stopping container nginx
2026/09/18 13:04:12   13:00:57 INFO Pod/agent-ops-ui-57b4fbdf48-p9qsd: Killing - Stopping container agent-ops-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container core-bff
2026/09/18 13:04:12   13:00:57 INFO Pod/gen-ai-ui-6fdf4d9489-m7jp5: Killing - Stopping container gen-ai-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/dashboard-redirect-5c78f45f7-zblf4: Killing - Stopping container nginx
2026/09/18 13:04:12   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:12   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:12   13:00:57 INFO Pod/eval-hub-ui-65bccdf794-994bs: Killing - Stopping container eval-hub-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/mlflow-ui-688cdfc884-4f94t: Killing - Stopping container mlflow-ui
2026/09/18 13:04:12   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:12   13:00:54 WARN Pod/kuberay-operator-6b5644f98d-dz8wl: Unhealthy - Readiness probe failed: Get "http://10.128.15.123:8082/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/18 13:04:12   13:00:53 INFO Pod/kuberay-operator-6b5644f98d-dz8wl: Killing - Stopping container kuberay-operator
2026/09/18 13:04:12   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-s7vxx to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: AddedInterface - Add eth0 [10.128.15.194/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Created - Created container: training-operator
2026/09/18 13:04:12   13:00:52 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 0 to 1
2026/09/18 13:04:12   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:04:12   13:00:52 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-s7vxx
2026/09/18 13:04:12   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Started - Started container training-operator
2026/09/18 13:04:12   13:00:50 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:50 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Killing - Stopping container training-operator
2026/09/18 13:04:12   13:00:50 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Killing - Stopping container training-operator
2026/09/18 13:04:12   13:00:49 INFO Pod/workbenches-operator-67897b697-2s98l: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:46 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:46 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:46 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:45 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:45 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Started - Started container manager
2026/09/18 13:04:12   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Created - Created container: manager
2026/09/18 13:04:12   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" in 4.377s (4.377s including waiting). Image size: 178385443 bytes.
2026/09/18 13:04:12   13:00:45 INFO Pod/ai-gateway-operator-57f8cd74c7-f8mps: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Created - Created container: manager
2026/09/18 13:04:12   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Started - Started container manager
2026/09/18 13:04:12   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:c34fec8cac5b49d18d6cd3318d5ebeac91df6e177b0c522a47c1cf0057b008bd" in 1.319s (1.319s including waiting). Image size: 193864580 bytes.
2026/09/18 13:04:12   13:00:41 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: AddedInterface - Add eth0 [10.128.15.193/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:41 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b"
2026/09/18 13:04:12   13:00:40 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:40 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/18 13:04:12   13:00:40 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 1 to 2
2026/09/18 13:04:12   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:c34fec8cac5b49d18d6cd3318d5ebeac91df6e177b0c522a47c1cf0057b008bd"
2026/09/18 13:04:12   13:00:40 INFO ReplicaSet/ogx-k8s-operator-controller-manager-6cbbbfbc9d: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw
2026/09/18 13:04:12   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: AddedInterface - Add eth0 [10.128.15.192/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:40 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-6cbbbfbc9d from 0 to 1
2026/09/18 13:04:12   13:00:40 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57t8kzt
2026/09/18 13:04:12   13:00:39 INFO Pod/maas-controller-f9dfb44dc-245mz: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:37 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-psfpn_b4e0b221-880c-4a02-89f8-334941ced849 became leader
2026/09/18 13:04:12   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f" in 1.285s (1.285s including waiting). Image size: 221650153 bytes.
2026/09/18 13:04:12   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Started - Started container automl-ui
2026/09/18 13:04:12   13:00:36 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-7fb7b967bd-pr77w_35674a84-aec5-4ff3-a77a-fbc41bcfd38c became leader
2026/09/18 13:04:12   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025" in 1.359s (1.359s including waiting). Image size: 203709694 bytes.
2026/09/18 13:04:12   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Created - Created container: autorag-ui
2026/09/18 13:04:12   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Created - Created container: automl-ui
2026/09/18 13:04:12   13:00:36 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6cc5fc4bb7-gfcxq_4c4bce57-75c4-4029-a42e-4b26570680cd became leader
2026/09/18 13:04:12   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Started - Started container autorag-ui
2026/09/18 13:04:12   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container core-bff
2026/09/18 13:04:12   13:00:35 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-dd84489b7 from 1 to 0
2026/09/18 13:04:12   13:00:35 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: AddedInterface - Add eth0 [10.129.45.162/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:35 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulDelete - Deleted pod: rhods-dashboard-dd84489b7-xsh2r
2026/09/18 13:04:12   13:00:35 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025"
2026/09/18 13:04:12   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:04:12   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container rhods-dashboard
2026/09/18 13:04:12   13:00:34 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-78c4c87bbf-w9xp6 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-dfdfcc946-24sv7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:34 INFO ReplicaSet/automl-ui-dfdfcc946: SuccessfulCreate - Created pod: automl-ui-dfdfcc946-24sv7
2026/09/18 13:04:12   13:00:34 INFO ReplicaSet/autorag-ui-78c4c87bbf: SuccessfulCreate - Created pod: autorag-ui-78c4c87bbf-w9xp6
2026/09/18 13:04:12   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f"
2026/09/18 13:04:12   13:00:34 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-679df8947-v85j2_2bc2d5b9-e8d2-4ca8-873b-9ac2f17f6fab became leader
2026/09/18 13:04:12   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: AddedInterface - Add eth0 [10.129.45.161/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:34 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-78c4c87bbf from 0 to 1
2026/09/18 13:04:12   13:00:33 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-f9dfb44dc-245mz_325308aa-ba4d-4984-ad9e-980b798a7bb4 became leader
2026/09/18 13:04:12   13:00:33 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-dfdfcc946 from 0 to 1
2026/09/18 13:04:12   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: AddedInterface - Add eth0 [10.128.15.191/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:04:12   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Created - Created container: manager
2026/09/18 13:04:12   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Started - Started container manager
2026/09/18 13:04:12   13:00:29 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:29 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 1 to 2
2026/09/18 13:04:12   13:00:29 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-k48cp
2026/09/18 13:04:12   13:00:27 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.081s (1.081s including waiting). Image size: 179699994 bytes.
2026/09/18 13:04:12   13:00:27 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Started - Started container core-bff
2026/09/18 13:04:12   13:00:27 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Created - Created container: core-bff
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Started - Started container rhods-dashboard
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Created - Created container: rhods-dashboard
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.031s (1.031s including waiting). Image size: 824448782 bytes.
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Created - Created container: kube-rbac-proxy
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Started - Started container kube-rbac-proxy
2026/09/18 13:04:12   13:00:26 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:04:12   13:00:25 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:04:12   13:00:25 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: AddedInterface - Add eth0 [10.128.15.190/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:24 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-xsh2r to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:24 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Created - Created container: manager
2026/09/18 13:04:12   13:00:24 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Started - Started container manager
2026/09/18 13:04:12   13:00:24 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-xsh2r
2026/09/18 13:04:12   13:00:24 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Created - Created container: manager
2026/09/18 13:04:12   13:00:24 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 0 to 1
2026/09/18 13:04:12   13:00:24 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:3d7ad3abf865793bb91a53a2543248a03891a9a001d93512d22c59aaf552dace" in 1.14s (1.14s including waiting). Image size: 173556749 bytes.
2026/09/18 13:04:12   13:00:24 INFO Pod/autorag-ui-78c4c87bbf-x6425: Killing - Stopping container autorag-ui
2026/09/18 13:04:12   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Started - Started container manager
2026/09/18 13:04:12   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: AddedInterface - Add eth0 [10.129.45.160/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" already present on machine
2026/09/18 13:04:12   13:00:24 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Started - Started container manager
2026/09/18 13:04:12   13:00:24 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Created - Created container: manager
2026/09/18 13:04:12   13:00:24 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:7fb76908e8ea80e485f52f537beeb7551ce05efe82ed22f5eb1ea1de148389d5" in 1.264s (1.264s including waiting). Image size: 168377417 bytes.
2026/09/18 13:04:12   13:00:23 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57n45qj to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:23 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:23 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:23 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:23 INFO Pod/automl-ui-dfdfcc946-wxwmk: Killing - Stopping container automl-ui
2026/09/18 13:04:12   13:00:23 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57n45qj
2026/09/18 13:04:12   13:00:23 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 0 to 1
2026/09/18 13:04:12   13:00:23 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:3d7ad3abf865793bb91a53a2543248a03891a9a001d93512d22c59aaf552dace"
2026/09/18 13:04:12   13:00:23 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: AddedInterface - Add eth0 [10.129.45.159/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:22 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-87cdb86d9-979zj to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:22 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75574d88c6-v85v8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:22 INFO ReplicaSet/odh-notebook-controller-manager-87cdb86d9: SuccessfulCreate - Created pod: odh-notebook-controller-manager-87cdb86d9-979zj
2026/09/18 13:04:12   13:00:22 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n_ddb2a1d2-3cea-4524-89ad-775ca39dde40 became leader
2026/09/18 13:04:12   13:00:22 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: AddedInterface - Add eth0 [10.128.15.189/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:22 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:7fb76908e8ea80e485f52f537beeb7551ce05efe82ed22f5eb1ea1de148389d5"
2026/09/18 13:04:12   13:00:22 INFO ReplicaSet/notebook-controller-deployment-75574d88c6: SuccessfulCreate - Created pod: notebook-controller-deployment-75574d88c6-v85v8
2026/09/18 13:04:12   13:00:22 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75574d88c6 from 0 to 1
2026/09/18 13:04:12   13:00:22 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Started - Started container manager
2026/09/18 13:04:12   13:00:22 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Created - Created container: manager
2026/09/18 13:04:12   13:00:22 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:90ab74599ab1294fa3e54bd0db517edff2df82b076b103f69b3ef791a75bf9dc" in 5.516s (5.516s including waiting). Image size: 186584994 bytes.
2026/09/18 13:04:12   13:00:22 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-87cdb86d9 from 0 to 1
2026/09/18 13:04:12   13:00:21 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-65d74dc6fc-f894w_669f6d03-4c4b-4d55-89be-4d7963abc46a became leader
2026/09/18 13:04:12   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Created - Created container: training-operator
2026/09/18 13:04:12   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:04:12   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: AddedInterface - Add eth0 [10.128.15.188/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:20 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Started - Started container training-operator
2026/09/18 13:04:12   13:00:19 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-4gvd8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:19 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 1 to 2
2026/09/18 13:04:12   13:00:19 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1.035s (1.035s including waiting). Image size: 238841358 bytes.
2026/09/18 13:04:12   13:00:19 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Created - Created container: manager
2026/09/18 13:04:12   13:00:19 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578" in 1.016s (1.016s including waiting). Image size: 194928572 bytes.
2026/09/18 13:04:12   13:00:19 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Started - Started container manager
2026/09/18 13:04:12   13:00:19 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Created - Created container: manager
2026/09/18 13:04:12   13:00:19 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-4gvd8
2026/09/18 13:04:12   13:00:19 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Started - Started container manager
2026/09/18 13:04:12   13:00:19 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Created - Created container: manager
2026/09/18 13:04:12   13:00:19 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Started - Started container manager
2026/09/18 13:04:12   13:00:18 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:18 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:04:12   13:00:18 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: AddedInterface - Add eth0 [10.128.15.187/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:18 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578"
2026/09/18 13:04:12   13:00:18 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 1.044s (1.044s including waiting). Image size: 238650146 bytes.
2026/09/18 13:04:12   13:00:18 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 1.054s (1.054s including waiting). Image size: 240120555 bytes.
2026/09/18 13:04:12   13:00:18 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Started - Started container manager
2026/09/18 13:04:12   13:00:18 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Created - Created container: manager
2026/09/18 13:04:12   13:00:17 INFO Pod/maas-controller-f9dfb44dc-245mz: Created - Created container: manager
2026/09/18 13:04:12   13:00:17 INFO Pod/maas-controller-f9dfb44dc-245mz: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:31870540df9132209a2865ddc3aeff7cc04ea3fdcb044c2e3b3ee18dcb73a688" in 3.304s (3.304s including waiting). Image size: 183271983 bytes.
2026/09/18 13:04:12   13:00:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: AddedInterface - Add eth0 [10.129.45.155/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:17 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Started - Started container copy-manifests
2026/09/18 13:04:12   13:00:17 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Created - Created container: copy-manifests
2026/09/18 13:04:12   13:00:17 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578" in 1.083s (1.083s including waiting). Image size: 194928572 bytes.
2026/09/18 13:04:12   13:00:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:04:12   13:00:17 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: AddedInterface - Add eth0 [10.129.45.156/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:17 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:04:12   13:00:17 INFO Pod/maas-controller-f9dfb44dc-245mz: Started - Started container manager
2026/09/18 13:04:12   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: AddedInterface - Add eth0 [10.129.45.158/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:04:12   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: Created - Created container: server
2026/09/18 13:04:12   13:00:17 INFO Pod/model-serving-api-7b65949446-lhg7k: Started - Started container server
2026/09/18 13:04:12   13:00:16 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-7fb7b967bd-pr77w to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:16 INFO Pod/model-serving-api-7b65949446-lhg7k: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-lhg7k to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:16 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-gfcxq to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:16 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:16 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-psfpn to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:16 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: AddedInterface - Add eth0 [10.129.45.153/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:16 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:a59ddd03e1fe14ad3f3f9d3505007d164a8aaa81a83e9bcad94c7b772c466578"
2026/09/18 13:04:12   13:00:16 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-lhg7k
2026/09/18 13:04:12   13:00:16 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-7fb7b967bd from 0 to 1
2026/09/18 13:04:12   13:00:16 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: AddedInterface - Add eth0 [10.129.45.154/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:16 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 0 to 1
2026/09/18 13:04:12   13:00:16 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:04:12   13:00:16 WARN Pod/model-serving-api-7b65949446-lhg7k: ErrorUpdatingResource - addLogicalPort failed for redhat-ods-applications/model-serving-api-7b65949446-lhg7k: failed to update pod redhat-ods-applications/model-serving-api-7b65949446-lhg7k: the server rejected our request due to an error in our request
2026/09/18 13:04:12   13:00:16 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-psfpn
2026/09/18 13:04:12   13:00:16 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:04:12   13:00:16 INFO ReplicaSet/odh-model-controller-7fb7b967bd: SuccessfulCreate - Created pod: odh-model-controller-7fb7b967bd-pr77w
2026/09/18 13:04:12   13:00:16 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:90ab74599ab1294fa3e54bd0db517edff2df82b076b103f69b3ef791a75bf9dc"
2026/09/18 13:04:12   13:00:16 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-gfcxq
2026/09/18 13:04:12   13:00:15 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:15 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:15 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:15 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-679df8947-v85j2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:15 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-7cfbff9c75: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n
2026/09/18 13:04:12   13:00:15 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-679df8947 from 0 to 1
2026/09/18 13:04:12   13:00:15 INFO ReplicaSet/opendatahub-ogx-operator-679df8947: SuccessfulCreate - Created pod: opendatahub-ogx-operator-679df8947-v85j2
2026/09/18 13:04:12   13:00:15 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-7cfbff9c75 from 0 to 1
2026/09/18 13:04:12   13:00:14 INFO Pod/workbenches-operator-67897b697-2s98l: Created - Created container: manager
2026/09/18 13:04:12   13:00:14 INFO Pod/workbenches-operator-67897b697-2s98l: Started - Started container manager
2026/09/18 13:04:12   13:00:14 INFO Pod/maas-controller-f9dfb44dc-245mz: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:31870540df9132209a2865ddc3aeff7cc04ea3fdcb044c2e3b3ee18dcb73a688"
2026/09/18 13:04:12   13:00:14 INFO Pod/maas-controller-f9dfb44dc-245mz: AddedInterface - Add eth0 [10.129.45.152/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:13 INFO Pod/maas-controller-f9dfb44dc-245mz: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-f9dfb44dc-245mz to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:13 INFO Pod/workbenches-operator-67897b697-2s98l: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:96c73966142a6cbf5a8b3dd69cbf92cae432987edde6a1cc39aa200915e7772c" already present on machine
2026/09/18 13:04:12   13:00:13 INFO ReplicaSet/maas-controller-f9dfb44dc: SuccessfulCreate - Created pod: maas-controller-f9dfb44dc-245mz
2026/09/18 13:04:12   13:00:13 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-f9dfb44dc from 0 to 1
2026/09/18 13:04:12   13:00:12 INFO Pod/workbenches-operator-67897b697-2s98l: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-67897b697-2s98l to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:12 INFO Pod/workbenches-operator-67897b697-2s98l: AddedInterface - Add eth0 [10.129.45.151/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:12 INFO ReplicaSet/workbenches-operator-67897b697: SuccessfulCreate - Created pod: workbenches-operator-67897b697-2s98l
2026/09/18 13:04:12   13:00:12 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-67897b697 from 0 to 1
2026/09/18 13:04:12   13:00:11 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Created - Created container: manager
2026/09/18 13:04:12   13:00:11 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Started - Started container manager
2026/09/18 13:04:12   13:00:11 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:fd4eae35eb2e80004d87d692bdf27f5bab20b0673011006307c0d54210934c84" in 4.497s (4.497s including waiting). Image size: 143072345 bytes.
2026/09/18 13:04:12   13:00:11 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8_966488f1-a994-4fa7-9de2-49eb9746e7bb became leader
2026/09/18 13:04:12   13:00:10 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:10 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57hbm9s: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:08 INFO Pod/opendatahub-feast-operator-5f45cd9467-nq89j: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:07 INFO Pod/opendatahub-ogx-operator-679df8947-4r8z2: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:06 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:06 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-bdc596ccc: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8
2026/09/18 13:04:12   13:00:06 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:fd4eae35eb2e80004d87d692bdf27f5bab20b0673011006307c0d54210934c84"
2026/09/18 13:04:12   13:00:06 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: AddedInterface - Add eth0 [10.129.45.150/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:06 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-bdc596ccc from 0 to 1
2026/09/18 13:04:12   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: AddedInterface - Add eth0 [10.129.45.149/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Started - Started container manager
2026/09/18 13:04:12   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Created - Created container: manager
2026/09/18 13:04:12   13:00:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:04:12   13:00:04 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:04 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-f894w to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   13:00:04 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 0 to 1
2026/09/18 13:04:12   13:00:04 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-f894w
2026/09/18 13:04:12   13:00:03 INFO Pod/kserve-controller-manager-75b9f799cd-j7q2h: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:03 INFO Pod/feast-operator-controller-manager-8fbf6785c-rpxbg: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:03 INFO Pod/odh-model-controller-7fb7b967bd-mj2g2: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:03 INFO Pod/model-serving-api-7b65949446-65hzr: Killing - Stopping container server
2026/09/18 13:04:12   13:00:03 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-5xd7k: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:02 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:02 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   13:00:00 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-g6chf: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:00 WARN Pod/workbenches-operator-67897b697-6dk8j: Unhealthy - Readiness probe failed: Get "http://10.128.15.121:8081/readyz": dial tcp 10.128.15.121:8081: connect: connection refused
2026/09/18 13:04:12   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: AddedInterface - Add eth0 [10.129.45.147/23] from ovn-kubernetes
2026/09/18 13:04:12   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:04:12   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Created - Created container: training-operator
2026/09/18 13:04:12   13:00:00 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Started - Started container training-operator
2026/09/18 13:04:12   13:00:00 INFO Pod/workbenches-operator-67897b697-6dk8j: Killing - Stopping container manager
2026/09/18 13:04:12   13:00:00 INFO Pod/maas-controller-f9dfb44dc-pc2jm: Killing - Stopping container manager
2026/09/18 13:04:12   12:59:59 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-dclv2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:04:12   12:59:59 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-dclv2
2026/09/18 13:04:12   12:59:58 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   12:59:58 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   12:59:58 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 0 to 1
2026/09/18 13:04:12   12:59:53 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-2r8qh: Killing - Stopping container manager
2026/09/18 13:04:12   12:59:52 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   12:59:52 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:04:12   12:59:49 INFO Pod/notebook-controller-deployment-75574d88c6-zkdn5: Killing - Stopping container manager
2026/09/18 13:04:12   12:59:49 INFO Pod/odh-notebook-controller-manager-87cdb86d9-s99zz: Killing - Stopping container manager
2026/09/18 13:04:12 === RESOURCE QUOTAS ===
2026/09/18 13:04:12 No resource quotas found in monitored namespaces
2026/09/18 13:04:12 Classification for Validate resource deletion recovery: infrastructure/storage (code=1006, confidence=medium)
2026/09/18 13:04:12   Evidence: event Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "eval-hub-proxy-tls" not found
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"storage","error_code":1006,"evidence":["event Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume \"proxy-tls\" : secret \"eval-hub-proxy-tls\" not found"],"confidence":"medium"}
2026/09/18 13:04:12 Diagnostics complete for failed test: Validate resource deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/18 13:05:27 TEST FAILURE DETECTED: group 1
2026/09/18 13:05:27 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 1) ===
2026/09/18 13:05:28 === CLUSTER STATE ===
2026/09/18 13:05:28 Node: ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:05:28   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:05:28   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:05:28   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:05:28   Resources: 15500m CPU, 63871024Ki memory allocatable (of 16000m CPU, 65022000Ki memory capacity)
2026/09/18 13:05:28 Node: ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:05:28   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:05:28   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:05:28   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:05:28   Resources: 15500m CPU, 63182884Ki memory allocatable (of 16000m CPU, 64333860Ki memory capacity)
2026/09/18 13:05:28 === DEPLOYMENTS ===
2026/09/18 13:05:28 Namespace: kube-system
2026/09/18 13:05:28 Namespace: redhat-ods-applications
2026/09/18 13:05:28   Deployment agent-ops-ui: 1/1 (READY)
2026/09/18 13:05:28   Deployment dashboard-operator: 2/2 (READY)
2026/09/18 13:05:28   Deployment dashboard-redirect: 2/2 (READY)
2026/09/18 13:05:28   Deployment eval-hub-ui: 1/1 (READY)
2026/09/18 13:05:28   Deployment gen-ai-ui: 1/1 (READY)
2026/09/18 13:05:28   Deployment maas-ui: 1/1 (READY)
2026/09/18 13:05:28   Deployment mlflow-operator-controller-manager: 1/1 (READY)
2026/09/18 13:05:28   Deployment mlflow-ui: 1/1 (READY)
2026/09/18 13:05:28   Deployment model-registry-operator-controller-manager: 1/1 (READY)
2026/09/18 13:05:28   Deployment model-registry-ui: 1/1 (READY)
2026/09/18 13:05:28   Deployment postgres-deployment: 1/1 (READY)
2026/09/18 13:05:28   Deployment rhods-dashboard: 2/2 (READY)
2026/09/18 13:05:28   Deployment spark-operator-controller: 1/1 (READY)
2026/09/18 13:05:28   Deployment spark-operator-webhook: 1/1 (READY)
2026/09/18 13:05:28   Deployment trustyai-service-operator-controller-manager: 1/1 (READY)
2026/09/18 13:05:28 === PODS ===
2026/09/18 13:05:28 Namespace redhat-ods-applications: no problematic pods found
2026/09/18 13:05:28 Namespace kube-system: no problematic pods found
2026/09/18 13:05:28 === OPERATOR STATUS ===
2026/09/18 13:05:28 Operator deployment: 3/3 ready
2026/09/18 13:05:28 === DSCI STATUS ===
2026/09/18 13:05:28 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; ThanosQuerierAvailable=False; PersesAvailable=False; NodeMetricsEndpointAvailable=False
2026/09/18 13:05:28 === DSC STATUS ===
2026/09/18 13:05:28 DSC default-dsc:
2026/09/18 13:05:28   Ready: True - 
2026/09/18 13:05:28   ProvisioningSucceeded: True - 
2026/09/18 13:05:28   ComponentsReady: True - 
2026/09/18 13:05:28   ModulesReady: True - 
2026/09/18 13:05:28   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/18 13:05:28   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/18 13:05:28   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/18 13:05:28   DashboardReady: True - 
2026/09/18 13:05:28   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/18 13:05:28   KserveLLMInferenceServiceDependencies: False - Submodule ManagementState is set to Removed
2026/09/18 13:05:28   KserveLLMInferenceServiceWideEPDependencies: False - Submodule ManagementState is set to Removed
2026/09/18 13:05:28   KserveReady: False - Module ManagementState is set to Removed
2026/09/18 13:05:28   KueueReady: True - 
2026/09/18 13:05:28   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/18 13:05:28   MLflowOperatorReady: True - MLflowOperator is ready to manage MLflow custom resources
2026/09/18 13:05:28   ModelRegistryReady: True - 
2026/09/18 13:05:28   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/18 13:05:28   OGXReady: False - Module ManagementState is set to Removed
2026/09/18 13:05:28   ProvisioningProgress: True - 
2026/09/18 13:05:28   RayReady: False - Component ManagementState is set to Removed
2026/09/18 13:05:28   SparkOperatorReady: True - 
2026/09/18 13:05:28   TrainerReady: False - Component ManagementState is set to Removed
2026/09/18 13:05:28   TrainingOperatorReady: False - Component ManagementState is set to Removed
2026/09/18 13:05:28   TrustyAIReady: True - 
2026/09/18 13:05:28   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/18 13:05:28 === RECENT EVENTS ===
2026/09/18 13:05:28   13:05:27 WARN Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Unhealthy - Readiness probe failed: Get "http://10.129.45.178:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/18 13:05:28   13:05:22 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Killing - Stopping container manager
2026/09/18 13:05:28   13:05:22 INFO Pod/kserve-module-controller-manager-6bb876bdfd-dn6pj: Killing - Stopping container manager
2026/09/18 13:05:28   13:05:22 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: Killing - Stopping container manager
2026/09/18 13:05:28   13:05:22 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Killing - Stopping container manager
2026/09/18 13:05:28   13:05:22 INFO Pod/model-serving-api-7b65949446-grjp5: Killing - Stopping container server
2026/09/18 13:05:28   13:05:21 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:05:21 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:05:04 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: Started - Started container manager
2026/09/18 13:05:28   13:05:04 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: Created - Created container: manager
2026/09/18 13:05:28   13:05:04 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 926ms (926ms including waiting). Image size: 238841358 bytes.
2026/09/18 13:05:28   13:05:03 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:05:28   13:05:03 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: AddedInterface - Add eth0 [10.129.45.180/23] from ovn-kubernetes
2026/09/18 13:05:28   13:05:02 INFO Pod/odh-model-controller-7fb7b967bd-xkrxz: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-7fb7b967bd-xkrxz to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:05:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Killing - Stopping container manager
2026/09/18 13:05:28   13:05:02 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-7fb7b967bd from 0 to 1
2026/09/18 13:05:28   13:05:02 INFO ReplicaSet/odh-model-controller-7fb7b967bd: SuccessfulCreate - Created pod: odh-model-controller-7fb7b967bd-xkrxz
2026/09/18 13:05:28   13:05:02 INFO Pod/odh-model-controller-7fb7b967bd-pr77w: Killing - Stopping container manager
2026/09/18 13:05:28   13:04:49 INFO Pod/model-serving-api-7b65949446-grjp5: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-grjp5 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:04:49 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-grjp5
2026/09/18 13:05:28   13:04:49 INFO Pod/model-serving-api-7b65949446-grjp5: AddedInterface - Add eth0 [10.129.45.179/23] from ovn-kubernetes
2026/09/18 13:05:28   13:04:49 INFO Pod/model-serving-api-7b65949446-grjp5: Created - Created container: server
2026/09/18 13:05:28   13:04:49 INFO Pod/model-serving-api-7b65949446-grjp5: Started - Started container server
2026/09/18 13:05:28   13:04:49 INFO Pod/model-serving-api-7b65949446-grjp5: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:05:28   13:04:49 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 0 to 1
2026/09/18 13:05:28   13:04:47 INFO Pod/model-serving-api-7b65949446-lhg7k: Killing - Stopping container server
2026/09/18 13:05:28   13:04:47 INFO Pod/model-serving-api-7b65949446-mjnc4: Killing - Stopping container server
2026/09/18 13:05:28   13:04:23 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6cc5fc4bb7-64g2q_18a0a5de-50be-492d-aae3-a645e21a236c became leader
2026/09/18 13:05:28   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 984ms (984ms including waiting). Image size: 238650146 bytes.
2026/09/18 13:05:28   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Created - Created container: manager
2026/09/18 13:05:28   13:04:05 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Started - Started container manager
2026/09/18 13:05:28   13:04:04 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:05:28   13:04:04 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: AddedInterface - Add eth0 [10.129.45.178/23] from ovn-kubernetes
2026/09/18 13:05:28   13:04:03 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-64g2q to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:04:03 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-64g2q
2026/09/18 13:05:28   13:04:03 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:05:28   13:04:02 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Killing - Stopping container manager
2026/09/18 13:05:28   13:04:02 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-gfcxq: Killing - Stopping container manager
2026/09/18 13:05:28   13:03:37 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-kr99f_83f4cae3-4c13-4eb1-a3ba-8afd6fc6cb0f became leader
2026/09/18 13:05:28   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Created - Created container: manager
2026/09/18 13:05:28   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 1.312s (1.312s including waiting). Image size: 240120555 bytes.
2026/09/18 13:05:28   13:03:20 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Started - Started container manager
2026/09/18 13:05:28   13:03:19 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: AddedInterface - Add eth0 [10.129.45.177/23] from ovn-kubernetes
2026/09/18 13:05:28   13:03:19 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:05:28   13:03:18 INFO Pod/kserve-controller-manager-75b9f799cd-kr99f: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-kr99f to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:03:18 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:05:28   13:03:18 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-kr99f
2026/09/18 13:05:28   13:03:17 INFO Pod/kserve-controller-manager-75b9f799cd-psfpn: Killing - Stopping container manager
2026/09/18 13:05:28   13:03:17 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Killing - Stopping container manager
2026/09/18 13:05:28   13:03:07 INFO ReplicaSet/rhods-dashboard-69fd7c8446: SuccessfulDelete - Deleted pod: rhods-dashboard-69fd7c8446-jw88c
2026/09/18 13:05:28   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container rhods-dashboard
2026/09/18 13:05:28   13:03:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-69fd7c8446 from 1 to 0
2026/09/18 13:05:28   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container core-bff
2026/09/18 13:05:28   13:03:07 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:05:28   13:02:31 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container core-bff
2026/09/18 13:05:28   13:02:31 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: core-bff
2026/09/18 13:05:28   13:02:30 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.081s (1.081s including waiting). Image size: 179699994 bytes.
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: kube-rbac-proxy
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container rhods-dashboard
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.205s (1.205s including waiting). Image size: 824448782 bytes.
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Started - Started container kube-rbac-proxy
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Created - Created container: rhods-dashboard
2026/09/18 13:05:28   13:02:29 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:05:28   13:02:28 INFO Pod/rhods-dashboard-dd84489b7-bxshp: AddedInterface - Add eth0 [10.129.45.176/23] from ovn-kubernetes
2026/09/18 13:05:28   13:02:28 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:05:28   13:02:27 INFO Pod/rhods-dashboard-dd84489b7-bxshp: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-bxshp to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:05:28   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container rhods-dashboard
2026/09/18 13:05:28   13:02:27 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Killing - Stopping container core-bff
2026/09/18 13:05:28   13:02:27 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-bxshp
2026/09/18 13:05:28   13:02:27 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulDelete - Deleted pod: rhods-dashboard-68569f858c-kcpdj
2026/09/18 13:05:28   13:02:27 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 1 to 2
2026/09/18 13:05:28   13:02:27 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-68569f858c from 1 to 0
2026/09/18 13:05:28   13:02:23 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Killing - Stopping container kuberay-operator
2026/09/18 13:05:28   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-x5nzf to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Created - Created container: kuberay-operator
2026/09/18 13:05:28   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:05:28   13:02:07 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 0 to 1
2026/09/18 13:05:28   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: AddedInterface - Add eth0 [10.129.45.175/23] from ovn-kubernetes
2026/09/18 13:05:28   13:02:07 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-x5nzf
2026/09/18 13:05:28   13:02:07 INFO Pod/kuberay-operator-6b5644f98d-x5nzf: Started - Started container kuberay-operator
2026/09/18 13:05:28   13:02:06 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Killing - Stopping container kuberay-operator
2026/09/18 13:05:28   13:02:06 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Killing - Stopping container kuberay-operator
2026/09/18 13:05:28   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1.054s (1.054s including waiting). Image size: 238841358 bytes.
2026/09/18 13:05:28   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Started - Started container manager
2026/09/18 13:05:28   13:02:02 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Created - Created container: manager
2026/09/18 13:05:28   13:02:01 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:05:28   13:02:01 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: AddedInterface - Add eth0 [10.129.45.174/23] from ovn-kubernetes
2026/09/18 13:05:28   13:02:00 INFO Pod/odh-model-controller-7fb7b967bd-2ntmx: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-7fb7b967bd-2ntmx to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:02:00 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-7fb7b967bd from 1 to 2
2026/09/18 13:05:28   13:02:00 INFO ReplicaSet/odh-model-controller-7fb7b967bd: SuccessfulCreate - Created pod: odh-model-controller-7fb7b967bd-2ntmx
2026/09/18 13:05:28   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:3b448f4f3d3ae31edf6bf16b807cfdae344acd64e7790ef6143b918298cd1432" already present on machine
2026/09/18 13:05:28   13:01:51 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container core-bff
2026/09/18 13:05:28   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Started - Started container manager
2026/09/18 13:05:28   13:01:51 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: core-bff
2026/09/18 13:05:28   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: AddedInterface - Add eth0 [10.128.15.206/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:51 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Created - Created container: manager
2026/09/18 13:05:28   13:01:50 INFO Pod/dashboard-operator-6f4c9f6fd5-bqf7t: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-6f4c9f6fd5-bqf7t to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:50 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-6f4c9f6fd5 from 1 to 2
2026/09/18 13:05:28   13:01:50 INFO ReplicaSet/dashboard-operator-6f4c9f6fd5: SuccessfulCreate - Created pod: dashboard-operator-6f4c9f6fd5-bqf7t
2026/09/18 13:05:28   13:01:50 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 2.363s (2.363s including waiting). Image size: 179699994 bytes.
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: rhods-dashboard
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container rhods-dashboard
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 927ms (927ms including waiting). Image size: 824448782 bytes.
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Started - Started container kube-rbac-proxy
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Created - Created container: kube-rbac-proxy
2026/09/18 13:05:28   13:01:48 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:05:28   13:01:47 INFO Pod/rhods-dashboard-dd84489b7-cnznb: AddedInterface - Add eth0 [10.128.15.205/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:47 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:05:28   13:01:46 INFO Pod/rhods-dashboard-dd84489b7-cnznb: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-dd84489b7-cnznb to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:46 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulDelete - Deleted pod: rhods-dashboard-68569f858c-9zbrr
2026/09/18 13:05:28   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container rhods-dashboard
2026/09/18 13:05:28   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container core-bff
2026/09/18 13:05:28   13:01:46 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:05:28   13:01:46 INFO Pod/automl-ui-dfdfcc946-27gns: Killing - Stopping container automl-ui
2026/09/18 13:05:28   13:01:46 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Killing - Stopping container autorag-ui
2026/09/18 13:05:28   13:01:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-dd84489b7 from 0 to 1
2026/09/18 13:05:28   13:01:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-68569f858c from 2 to 1
2026/09/18 13:05:28   13:01:46 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulCreate - Created pod: rhods-dashboard-dd84489b7-cnznb
2026/09/18 13:05:28   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:05:28   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Started - Started container server
2026/09/18 13:05:28   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: Created - Created container: server
2026/09/18 13:05:28   13:01:41 INFO Pod/model-serving-api-7b65949446-mjnc4: AddedInterface - Add eth0 [10.128.15.204/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:40 INFO Pod/model-serving-api-7b65949446-mjnc4: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-mjnc4 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:40 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-mjnc4
2026/09/18 13:05:28   13:01:40 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 1 to 2
2026/09/18 13:05:28   13:01:37 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:37 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Created - Created container: automl-ui
2026/09/18 13:05:28   13:01:37 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: core-bff
2026/09/18 13:05:28   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container core-bff
2026/09/18 13:05:28   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Started - Started container automl-ui
2026/09/18 13:05:28   13:01:37 INFO Pod/automl-ui-dfdfcc946-27gns: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f" in 1.451s (1.451s including waiting). Image size: 221650153 bytes.
2026/09/18 13:05:28   13:01:37 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.041s (1.041s including waiting). Image size: 179699994 bytes.
2026/09/18 13:05:28   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:43318525e1012c07e025c225f867db58de51c87eba65d0758fcf06af35cf3799" in 1.572s (1.572s including waiting). Image size: 246518882 bytes.
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: rhods-dashboard
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:05:28   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Started - Started container eval-hub-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/model-registry-ui-986875f9b-cqjtt: Started - Started container model-registry-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Created - Created container: eval-hub-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-mlflow-rhel9@sha256:72da9df1353bb583feeb9dee9afa1a4439fba5749bbaf2599c301c0009fc1cb4" in 1.643s (1.643s including waiting). Image size: 179358952 bytes.
2026/09/18 13:05:28   13:01:36 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-eval-hub-rhel9@sha256:e957c89fc6ff354d10a6b49abd63613ed7a82157cfe0c790b0fa07a883292e6a" in 1.491s (1.491s including waiting). Image size: 187473359 bytes.
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.15s (1.15s including waiting). Image size: 824448782 bytes.
2026/09/18 13:05:28   13:01:36 INFO Pod/model-registry-ui-986875f9b-cqjtt: Created - Created container: model-registry-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Started - Started container mlflow-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:05:28   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Created - Created container: gen-ai-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Created - Created container: mlflow-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container kube-rbac-proxy
2026/09/18 13:05:28   13:01:36 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Started - Started container gen-ai-ui
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Created - Created container: kube-rbac-proxy
2026/09/18 13:05:28   13:01:36 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Started - Started container rhods-dashboard
2026/09/18 13:05:28   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Created - Created container: agent-ops-ui
2026/09/18 13:05:28   13:01:35 INFO Pod/automl-ui-dfdfcc946-27gns: AddedInterface - Add eth0 [10.129.45.172/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:35 INFO Pod/automl-ui-dfdfcc946-27gns: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f"
2026/09/18 13:05:28   13:01:35 INFO Pod/model-registry-ui-986875f9b-cqjtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-model-registry-rhel9@sha256:9158b18f63124db7c53589ba1df2b5aee9a32db7465ec785ae37b11af3d049ea" in 1.25s (1.25s including waiting). Image size: 199026570 bytes.
2026/09/18 13:05:28   13:01:35 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:05:28   13:01:35 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: AddedInterface - Add eth0 [10.129.45.173/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:35 INFO Pod/eval-hub-ui-65bccdf794-m78fg: AddedInterface - Add eth0 [10.128.15.203/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Started - Started container agent-ops-ui
2026/09/18 13:05:28   13:01:35 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:5e2b0975d8c0be0163e244494f33289fb3e99791798e3b1b1b0fe6421497ee0b" in 1.478s (1.478s including waiting). Image size: 196978981 bytes.
2026/09/18 13:05:28   13:01:35 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-eval-hub-rhel9@sha256:e957c89fc6ff354d10a6b49abd63613ed7a82157cfe0c790b0fa07a883292e6a"
2026/09/18 13:05:28   13:01:34 INFO Pod/rhods-dashboard-69fd7c8446-jw88c: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-69fd7c8446-jw88c to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:34 INFO Pod/automl-ui-dfdfcc946-27gns: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-dfdfcc946-27gns to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:34 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: AddedInterface - Add eth0 [10.128.15.201/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:34 INFO Pod/mlflow-ui-688cdfc884-hfxw2: AddedInterface - Add eth0 [10.129.45.171/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:34 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:43318525e1012c07e025c225f867db58de51c87eba65d0758fcf06af35cf3799"
2026/09/18 13:05:28   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:03a66fd093e7ede531f749d1246d395b52c767bed49c6ff61000a50516137939" in 1.257s (1.257s including waiting). Image size: 189987009 bytes.
2026/09/18 13:05:28   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Created - Created container: maas-ui
2026/09/18 13:05:28   13:01:34 INFO Pod/maas-ui-778d4756c9-gfqbk: Started - Started container maas-ui
2026/09/18 13:05:28   13:01:34 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-dfdfcc946 from 0 to 1
2026/09/18 13:05:28   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container core-bff
2026/09/18 13:05:28   13:01:34 WARN Pod/eval-hub-ui-65bccdf794-m78fg: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "eval-hub-proxy-tls" not found
2026/09/18 13:05:28   13:01:34 INFO ReplicaSet/automl-ui-dfdfcc946: SuccessfulCreate - Created pod: automl-ui-dfdfcc946-27gns
2026/09/18 13:05:28   13:01:34 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-mlflow-rhel9@sha256:72da9df1353bb583feeb9dee9afa1a4439fba5749bbaf2599c301c0009fc1cb4"
2026/09/18 13:05:28   13:01:34 INFO Pod/model-registry-ui-986875f9b-cqjtt: AddedInterface - Add eth0 [10.128.15.202/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:34 INFO Pod/model-registry-ui-986875f9b-cqjtt: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-model-registry-rhel9@sha256:9158b18f63124db7c53589ba1df2b5aee9a32db7465ec785ae37b11af3d049ea"
2026/09/18 13:05:28   13:01:34 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: core-bff
2026/09/18 13:05:28   13:01:34 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container core-bff
2026/09/18 13:05:28   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: core-bff
2026/09/18 13:05:28   13:01:34 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 949ms (949ms including waiting). Image size: 179699994 bytes.
2026/09/18 13:05:28   13:01:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-69fd7c8446 from 0 to 1
2026/09/18 13:05:28   13:01:34 INFO ReplicaSet/rhods-dashboard-69fd7c8446: SuccessfulCreate - Created pod: rhods-dashboard-69fd7c8446-jw88c
2026/09/18 13:05:28   13:01:34 WARN Pod/automl-ui-dfdfcc946-27gns: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/18 13:05:28   13:01:33 INFO Pod/eval-hub-ui-65bccdf794-m78fg: Scheduled - Successfully assigned redhat-ods-applications/eval-hub-ui-65bccdf794-m78fg to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:33 INFO Pod/mlflow-ui-688cdfc884-hfxw2: Scheduled - Successfully assigned redhat-ods-applications/mlflow-ui-688cdfc884-hfxw2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:33 INFO Pod/model-registry-ui-986875f9b-cqjtt: Scheduled - Successfully assigned redhat-ods-applications/model-registry-ui-986875f9b-cqjtt to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:33 INFO ReplicaSet/model-registry-ui-986875f9b: SuccessfulCreate - Created pod: model-registry-ui-986875f9b-cqjtt
2026/09/18 13:05:28   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Started - Started container autorag-ui
2026/09/18 13:05:28   13:01:33 INFO Deployment/mlflow-ui: ScalingReplicaSet - Scaled up replica set mlflow-ui-688cdfc884 from 0 to 1
2026/09/18 13:05:28   13:01:33 INFO ReplicaSet/mlflow-ui-688cdfc884: SuccessfulCreate - Created pod: mlflow-ui-688cdfc884-hfxw2
2026/09/18 13:05:28   13:01:33 WARN Pod/mlflow-ui-688cdfc884-hfxw2: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "mlflow-proxy-tls" not found
2026/09/18 13:05:28   13:01:33 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.591s (1.591s including waiting). Image size: 179699994 bytes.
2026/09/18 13:05:28   13:01:33 INFO ReplicaSet/eval-hub-ui-65bccdf794: SuccessfulCreate - Created pod: eval-hub-ui-65bccdf794-m78fg
2026/09/18 13:05:28   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025" in 1.039s (1.039s including waiting). Image size: 203709694 bytes.
2026/09/18 13:05:28   13:01:33 INFO Deployment/eval-hub-ui: ScalingReplicaSet - Scaled up replica set eval-hub-ui-65bccdf794 from 0 to 1
2026/09/18 13:05:28   13:01:33 INFO Deployment/model-registry-ui: ScalingReplicaSet - Scaled up replica set model-registry-ui-986875f9b from 0 to 1
2026/09/18 13:05:28   13:01:33 WARN Pod/model-registry-ui-986875f9b-cqjtt: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "model-registry-proxy-tls" not found
2026/09/18 13:05:28   13:01:33 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:5e2b0975d8c0be0163e244494f33289fb3e99791798e3b1b1b0fe6421497ee0b"
2026/09/18 13:05:28   13:01:33 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Created - Created container: autorag-ui
2026/09/18 13:05:28   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:05:28   13:01:33 INFO Pod/maas-ui-778d4756c9-gfqbk: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:03a66fd093e7ede531f749d1246d395b52c767bed49c6ff61000a50516137939"
2026/09/18 13:05:28   13:01:33 INFO Pod/maas-ui-778d4756c9-gfqbk: AddedInterface - Add eth0 [10.129.45.169/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container kube-rbac-proxy
2026/09/18 13:05:28   13:01:33 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: kube-rbac-proxy
2026/09/18 13:05:28   13:01:33 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: AddedInterface - Add eth0 [10.129.45.170/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:32 INFO Pod/agent-ops-ui-57b4fbdf48-t6px6: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-57b4fbdf48-t6px6 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:32 INFO Pod/maas-ui-778d4756c9-gfqbk: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-778d4756c9-gfqbk to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:32 INFO Pod/gen-ai-ui-6fdf4d9489-cw8r8: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-6fdf4d9489-cw8r8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:32 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-6fdf4d9489 from 0 to 1
2026/09/18 13:05:28   13:01:32 INFO ReplicaSet/maas-ui-778d4756c9: SuccessfulCreate - Created pod: maas-ui-778d4756c9-gfqbk
2026/09/18 13:05:28   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Created - Created container: kuberay-operator
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:05:28   13:01:32 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-57b4fbdf48 from 0 to 1
2026/09/18 13:05:28   13:01:32 INFO ReplicaSet/agent-ops-ui-57b4fbdf48: SuccessfulCreate - Created pod: agent-ops-ui-57b4fbdf48-t6px6
2026/09/18 13:05:28   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:05:28   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Started - Started container kuberay-operator
2026/09/18 13:05:28   13:01:32 INFO ReplicaSet/gen-ai-ui-6fdf4d9489: SuccessfulCreate - Created pod: gen-ai-ui-6fdf4d9489-cw8r8
2026/09/18 13:05:28   13:01:32 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: AddedInterface - Add eth0 [10.129.45.168/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Started - Started container rhods-dashboard
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Created - Created container: rhods-dashboard
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.458s (1.458s including waiting). Image size: 824448782 bytes.
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container kube-rbac-proxy
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.014s (1.014s including waiting). Image size: 824448782 bytes.
2026/09/18 13:05:28   13:01:32 WARN Pod/gen-ai-ui-6fdf4d9489-cw8r8: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "gen-ai-proxy-tls" not found
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: kube-rbac-proxy
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Started - Started container rhods-dashboard
2026/09/18 13:05:28   13:01:32 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-778d4756c9 from 0 to 1
2026/09/18 13:05:28   13:01:32 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:32 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Created - Created container: rhods-dashboard
2026/09/18 13:05:28   13:01:32 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025"
2026/09/18 13:05:28   13:01:32 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: AddedInterface - Add eth0 [10.128.15.200/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:31 INFO Pod/autorag-ui-78c4c87bbf-t8j5f: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-78c4c87bbf-t8j5f to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:31 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:31 INFO Pod/kuberay-operator-6b5644f98d-2c7hq: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-2c7hq to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:31 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-2c7hq
2026/09/18 13:05:28   13:01:31 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 1 to 2
2026/09/18 13:05:28   13:01:31 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:05:28   13:01:31 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-6b5644f98d-9x9df_8b0c9b3f-bc3c-4fe4-8ca3-2ebfe112f55f became leader
2026/09/18 13:05:28   13:01:31 INFO Pod/rhods-dashboard-68569f858c-9zbrr: AddedInterface - Add eth0 [10.129.45.167/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:31 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:05:28   13:01:31 INFO Pod/rhods-dashboard-68569f858c-kcpdj: AddedInterface - Add eth0 [10.128.15.199/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:31 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-78c4c87bbf from 0 to 1
2026/09/18 13:05:28   13:01:31 INFO ReplicaSet/autorag-ui-78c4c87bbf: SuccessfulCreate - Created pod: autorag-ui-78c4c87bbf-t8j5f
2026/09/18 13:05:28   13:01:30 INFO Pod/rhods-dashboard-68569f858c-9zbrr: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-68569f858c-9zbrr to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:30 INFO Pod/rhods-dashboard-68569f858c-kcpdj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-68569f858c-kcpdj to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:30 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/18 13:05:28   13:01:30 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-68569f858c from 0 to 2
2026/09/18 13:05:28   13:01:30 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulCreate - Created pod: rhods-dashboard-68569f858c-kcpdj
2026/09/18 13:05:28   13:01:30 INFO ReplicaSet/rhods-dashboard-68569f858c: SuccessfulCreate - Created pod: rhods-dashboard-68569f858c-9zbrr
2026/09/18 13:05:28   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Created - Created container: manager
2026/09/18 13:05:28   13:01:26 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-6f4c9f6fd5-v5llf_d0181e54-6ac3-41a5-beef-01d007de8627 became leader
2026/09/18 13:05:28   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 4.976s (4.976s including waiting). Image size: 238650146 bytes.
2026/09/18 13:05:28   13:01:26 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Started - Started container manager
2026/09/18 13:05:28   13:01:23 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9_8a31aa91-6e9d-4828-9032-14144f389211 became leader
2026/09/18 13:05:28   13:01:21 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:05:28   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Created - Created container: manager
2026/09/18 13:05:28   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Started - Started container manager
2026/09/18 13:05:28   13:01:21 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: AddedInterface - Add eth0 [10.128.15.198/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" already present on machine
2026/09/18 13:05:28   13:01:21 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: AddedInterface - Add eth0 [10.129.45.166/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57lbbcr to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:20 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-plxn8: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-plxn8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:20 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 1 to 2
2026/09/18 13:05:28   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57n45qj: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:20 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:20 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 0 to 1
2026/09/18 13:05:28   13:01:20 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-plxn8
2026/09/18 13:05:28   13:01:20 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57lbbcr
2026/09/18 13:05:28   13:01:17 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:17 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:16 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:01:16 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Killing - Stopping container training-operator
2026/09/18 13:05:28   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: AddedInterface - Add eth0 [10.128.15.197/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:334da90acc2d06efa954018e0c89bd253bad3b2bda2b1ea18dc6ff3ae23945b4" already present on machine
2026/09/18 13:05:28   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Started - Started container kuberay-operator
2026/09/18 13:05:28   13:01:13 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Created - Created container: kuberay-operator
2026/09/18 13:05:28   13:01:13 WARN Pod/rhods-dashboard-69fd7c8446-7z84q: Unhealthy - Readiness probe failed: Get "https://10.129.45.22:8943/healthcheck": context deadline exceeded
2026/09/18 13:05:28   13:01:12 INFO Pod/kuberay-operator-6b5644f98d-9x9df: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-6b5644f98d-9x9df to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:12 INFO ReplicaSet/kuberay-operator-6b5644f98d: SuccessfulCreate - Created pod: kuberay-operator-6b5644f98d-9x9df
2026/09/18 13:05:28   13:01:12 WARN Pod/kuberay-operator-6b5644f98d-9x9df: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/18 13:05:28   13:01:12 INFO Pod/opendatahub-ogx-operator-679df8947-v85j2: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:12 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-6b5644f98d from 0 to 1
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:a61265350ee8404e4792cd06dd4f1acc7236b5b5c90bff4c543ee6ac02ab4cfd" already present on machine
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: AddedInterface - Add eth0 [10.128.15.196/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Created - Created container: nginx
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Created - Created container: nginx
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:a61265350ee8404e4792cd06dd4f1acc7236b5b5c90bff4c543ee6ac02ab4cfd" already present on machine
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Started - Started container nginx
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Started - Started container nginx
2026/09/18 13:05:28   13:01:11 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: AddedInterface - Add eth0 [10.129.45.165/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:10 INFO Pod/dashboard-redirect-5c78f45f7-hc4m7: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-5c78f45f7-hc4m7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:10 INFO Pod/dashboard-redirect-5c78f45f7-kvxnc: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-5c78f45f7-kvxnc to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Created - Created container: manager
2026/09/18 13:05:28   13:01:10 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-5c78f45f7 from 0 to 2
2026/09/18 13:05:28   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Started - Started container manager
2026/09/18 13:05:28   13:01:10 INFO ReplicaSet/dashboard-redirect-5c78f45f7: SuccessfulCreate - Created pod: dashboard-redirect-5c78f45f7-hc4m7
2026/09/18 13:05:28   13:01:10 INFO ReplicaSet/dashboard-redirect-5c78f45f7: SuccessfulCreate - Created pod: dashboard-redirect-5c78f45f7-kvxnc
2026/09/18 13:05:28   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: AddedInterface - Add eth0 [10.129.45.164/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:10 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:3b448f4f3d3ae31edf6bf16b807cfdae344acd64e7790ef6143b918298cd1432" already present on machine
2026/09/18 13:05:28   13:01:09 INFO Pod/dashboard-operator-6f4c9f6fd5-v5llf: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-6f4c9f6fd5-v5llf to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:09 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-6f4c9f6fd5 from 0 to 1
2026/09/18 13:05:28   13:01:09 INFO ReplicaSet/dashboard-operator-6f4c9f6fd5: SuccessfulCreate - Created pod: dashboard-operator-6f4c9f6fd5-v5llf
2026/09/18 13:05:28   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: AddedInterface - Add eth0 [10.129.45.163/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Started - Started container manager
2026/09/18 13:05:28   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:05:28   13:01:06 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Created - Created container: manager
2026/09/18 13:05:28   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:05 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 0 to 1
2026/09/18 13:05:28   13:01:05 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-zzqb9
2026/09/18 13:05:28   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:05 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-f894w: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 945ms (945ms including waiting). Image size: 240120555 bytes.
2026/09/18 13:05:28   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Started - Started container manager
2026/09/18 13:05:28   13:01:02 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Created - Created container: manager
2026/09/18 13:05:28   13:01:01 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Killing - Stopping container manager
2026/09/18 13:05:28   13:01:01 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: AddedInterface - Add eth0 [10.128.15.195/23] from ovn-kubernetes
2026/09/18 13:05:28   13:01:01 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:05:28   13:01:00 INFO Pod/kserve-controller-manager-75b9f799cd-qqvq8: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-qqvq8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:01:00 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-qqvq8
2026/09/18 13:05:28   13:01:00 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 1 to 2
2026/09/18 13:05:28   13:00:59 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw_5db8ff17-300e-4d97-a950-cc77ad40f5fd became leader
2026/09/18 13:05:28   13:00:59 INFO Pod/dashboard-operator-6f4c9f6fd5-bsxjp: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:58 WARN Pod/rhods-dashboard-69fd7c8446-7z84q: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = NotFound desc = container is not created or running: checking if PID of cb91b77b69001a0a7ddc3609e04c59c60afda371a164891689713f7b3b6a4abc is running failed: container process not found
2026/09/18 13:05:28   13:00:57 INFO Pod/dashboard-redirect-5c78f45f7-hp5th: Killing - Stopping container nginx
2026/09/18 13:05:28   13:00:57 INFO Pod/eval-hub-ui-65bccdf794-994bs: Killing - Stopping container eval-hub-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/agent-ops-ui-57b4fbdf48-p9qsd: Killing - Stopping container agent-ops-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container core-bff
2026/09/18 13:05:28   13:00:57 INFO Pod/mlflow-ui-688cdfc884-4f94t: Killing - Stopping container mlflow-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container rhods-dashboard
2026/09/18 13:05:28   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:05:28   13:00:57 INFO Pod/gen-ai-ui-6fdf4d9489-m7jp5: Killing - Stopping container gen-ai-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/dashboard-redirect-5c78f45f7-zblf4: Killing - Stopping container nginx
2026/09/18 13:05:28   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container rhods-dashboard
2026/09/18 13:05:28   13:00:57 INFO Pod/maas-ui-778d4756c9-vpwmd: Killing - Stopping container maas-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Killing - Stopping container autorag-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/automl-ui-dfdfcc946-24sv7: Killing - Stopping container automl-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-7z84q: Killing - Stopping container core-bff
2026/09/18 13:05:28   13:00:57 INFO Pod/model-registry-ui-986875f9b-h5wj4: Killing - Stopping container model-registry-ui
2026/09/18 13:05:28   13:00:57 INFO Pod/rhods-dashboard-69fd7c8446-vml99: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:05:28   13:00:54 WARN Pod/kuberay-operator-6b5644f98d-dz8wl: Unhealthy - Readiness probe failed: Get "http://10.128.15.123:8082/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/18 13:05:28   13:00:53 INFO Pod/kuberay-operator-6b5644f98d-dz8wl: Killing - Stopping container kuberay-operator
2026/09/18 13:05:28   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-training-operator-77857cf598-s7vxx to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Pulled - Container image "quay.io/rhoai/odh-training-operator-rhel9@sha256:28b0bf6902ae52a6487f6917da79dda14061e8b80a160b11bbdb7febb9a5f89a" already present on machine
2026/09/18 13:05:28   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Started - Started container training-operator
2026/09/18 13:05:28   13:00:52 INFO ReplicaSet/kubeflow-training-operator-77857cf598: SuccessfulCreate - Created pod: kubeflow-training-operator-77857cf598-s7vxx
2026/09/18 13:05:28   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: AddedInterface - Add eth0 [10.128.15.194/23] from ovn-kubernetes
2026/09/18 13:05:28   13:00:52 INFO Deployment/kubeflow-training-operator: ScalingReplicaSet - Scaled up replica set kubeflow-training-operator-77857cf598 from 0 to 1
2026/09/18 13:05:28   13:00:52 INFO Pod/kubeflow-training-operator-77857cf598-s7vxx: Created - Created container: training-operator
2026/09/18 13:05:28   13:00:50 INFO Pod/kubeflow-training-operator-77857cf598-4gvd8: Killing - Stopping container training-operator
2026/09/18 13:05:28   13:00:50 INFO Pod/kubeflow-training-operator-77857cf598-dclv2: Killing - Stopping container training-operator
2026/09/18 13:05:28   13:00:50 INFO Pod/mcp-lifecycle-module-operator-controller-manager-bdc596ccc4f5k8: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:49 INFO Pod/workbenches-operator-67897b697-2s98l: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:46 INFO Pod/mcp-lifecycle-operator-controller-manager-7cfbff9c75-mvt9n: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:46 INFO Pod/odh-notebook-controller-manager-87cdb86d9-979zj: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:46 INFO Pod/notebook-controller-deployment-75574d88c6-v85v8: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:45 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:00:45 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:05:28   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Started - Started container manager
2026/09/18 13:05:28   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b" in 4.377s (4.377s including waiting). Image size: 178385443 bytes.
2026/09/18 13:05:28   13:00:45 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Created - Created container: manager
2026/09/18 13:05:28   13:00:45 INFO Pod/ai-gateway-operator-57f8cd74c7-f8mps: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:c34fec8cac5b49d18d6cd3318d5ebeac91df6e177b0c522a47c1cf0057b008bd" in 1.319s (1.319s including waiting). Image size: 193864580 bytes.
2026/09/18 13:05:28   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Created - Created container: manager
2026/09/18 13:05:28   13:00:42 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Started - Started container manager
2026/09/18 13:05:28   13:00:41 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:797f99c4d1a4ba3d35c89e54662dd6a45e51eb10b7234b09a36581a8a33eae3b"
2026/09/18 13:05:28   13:00:41 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: AddedInterface - Add eth0 [10.128.15.193/23] from ovn-kubernetes
2026/09/18 13:05:28   13:00:40 INFO Pod/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-7bf5f57t8kzt to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:00:40 INFO ReplicaSet/ogx-k8s-operator-controller-manager-6cbbbfbc9d: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw
2026/09/18 13:05:28   13:00:40 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-7bf5f57b8b: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-7bf5f57t8kzt
2026/09/18 13:05:28   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: AddedInterface - Add eth0 [10.128.15.192/23] from ovn-kubernetes
2026/09/18 13:05:28   13:00:40 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-7bf5f57b8b from 1 to 2
2026/09/18 13:05:28   13:00:40 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-6cbbbfbc9d from 0 to 1
2026/09/18 13:05:28   13:00:40 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/18 13:05:28   13:00:40 INFO Pod/ogx-k8s-operator-controller-manager-6cbbbfbc9d-zw8fw: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:c34fec8cac5b49d18d6cd3318d5ebeac91df6e177b0c522a47c1cf0057b008bd"
2026/09/18 13:05:28   13:00:39 INFO Pod/maas-controller-f9dfb44dc-245mz: Killing - Stopping container manager
2026/09/18 13:05:28   13:00:37 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-psfpn_b4e0b221-880c-4a02-89f8-334941ced849 became leader
2026/09/18 13:05:28   13:00:36 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-7fb7b967bd-pr77w_35674a84-aec5-4ff3-a77a-fbc41bcfd38c became leader
2026/09/18 13:05:28   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025" in 1.359s (1.359s including waiting). Image size: 203709694 bytes.
2026/09/18 13:05:28   13:00:36 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6cc5fc4bb7-gfcxq_4c4bce57-75c4-4029-a42e-4b26570680cd became leader
2026/09/18 13:05:28   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f" in 1.285s (1.285s including waiting). Image size: 221650153 bytes.
2026/09/18 13:05:28   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Created - Created container: automl-ui
2026/09/18 13:05:28   13:00:36 INFO Pod/automl-ui-dfdfcc946-24sv7: Started - Started container automl-ui
2026/09/18 13:05:28   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Started - Started container autorag-ui
2026/09/18 13:05:28   13:00:36 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Created - Created container: autorag-ui
2026/09/18 13:05:28   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container rhods-dashboard
2026/09/18 13:05:28   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container core-bff
2026/09/18 13:05:28   13:00:35 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: AddedInterface - Add eth0 [10.129.45.162/23] from ovn-kubernetes
2026/09/18 13:05:28   13:00:35 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-dd84489b7 from 1 to 0
2026/09/18 13:05:28   13:00:35 INFO ReplicaSet/rhods-dashboard-dd84489b7: SuccessfulDelete - Deleted pod: rhods-dashboard-dd84489b7-xsh2r
2026/09/18 13:05:28   13:00:35 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:24098c6c566fdd1bf6095674fa5d56e0fd2e42196040a799a830215399b19025"
2026/09/18 13:05:28   13:00:35 INFO Pod/rhods-dashboard-dd84489b7-xsh2r: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:05:28   13:00:34 INFO Pod/autorag-ui-78c4c87bbf-w9xp6: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-78c4c87bbf-w9xp6 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-dfdfcc946-24sv7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:05:28   13:00:34 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-78c4c87bbf from 0 to 1
2026/09/18 13:05:28   13:00:34 INFO ReplicaSet/automl-ui-dfdfcc946: SuccessfulCreate - Created pod: automl-ui-dfdfcc946-24sv7
2026/09/18 13:05:28   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: AddedInterface - Add eth0 [10.129.45.161/23] from ovn-kubernetes
2026/09/18 13:05:28   13:00:34 INFO ReplicaSet/autorag-ui-78c4c87bbf: SuccessfulCreate - Created pod: autorag-ui-78c4c87bbf-w9xp6
2026/09/18 13:05:28   13:00:34 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-679df8947-v85j2_2bc2d5b9-e8d2-4ca8-873b-9ac2f17f6fab became leader
2026/09/18 13:05:28   13:00:34 INFO Pod/automl-ui-dfdfcc946-24sv7: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:a03d6fdf83f0adc3080413fa7c0f0ff58fa8f170128636be9cff7605e9e2535f"
2026/09/18 13:05:28   13:00:33 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-f9dfb44dc-245mz_325308aa-ba4d-4984-ad9e-980b798a7bb4 became leader
2026/09/18 13:05:28   13:00:33 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-dfdfcc946 from 0 to 1
2026/09/18 13:05:28   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Created - Created container: manager
2026/09/18 13:05:28   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: AddedInterface - Add eth0 [10.128.15.191/23] from ovn-kubernetes
2026/09/18 13:05:28   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:05:28   13:00:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Started - Started container manager
2026/09/18 13:05:28   13:00:29 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-k48cp to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:05:28   13:00:29 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 1 to 2
2026/09/18 13:05:28   13:00:29 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-k48cp
2026/09/18 13:05:28 === RESOURCE QUOTAS ===
2026/09/18 13:05:28 No resource quotas found in monitored namespaces
2026/09/18 13:05:28 Classification for group 1: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/18 13:05:28   Evidence: event Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Unhealthy - Readiness probe failed: Get "http://10.129.45.178:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/llmisvc-controller-manager-6cc5fc4bb7-64g2q: Unhealthy - Readiness probe failed: Get \"http://10.129.45.178:8081/readyz\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"],"confidence":"medium"}
2026/09/18 13:05:28 Diagnostics complete for failed test: group 1
    controller_test.go:767: Stopping: group 1 test failed.
2026/09/18 13:05:28 Circuit breaker: failure classified as infrastructure/probe-failure (code=1014)
2026/09/18 13:05:28 Circuit breaker: infrastructure/unknown failure recorded (1/3)
    helper_test.go:157: Verifying Kueue configuration (cluster) is still created.
    helper_test.go:157: Verifying no ClusterQueue, ResourceFlavor or LocalQueue is created (autoCreateQueues=false).
    helper_test.go:157: Cleaning up Kueue test resources.
    helper_test.go:157: Cleaning up Kueue cluster-scoped resources
    helper_test.go:157: Will attempt to delete 3 Kueue resources
    helper_test.go:157: Attempting to delete ClusterQueue /default
    helper_test.go:157: Successfully processed deletion of ClusterQueue /default
    helper_test.go:157: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:157: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:157: Attempting to delete Kueue /cluster
    helper_test.go:157: Successfully processed deletion of Kueue /cluster
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind'
2026-09-18T13:13:23Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.5.1"}, "Cluster": {"type":"OpenShift","version":"4.21.19"}}
2026/09/18 13:16:35 TEST FAILURE DETECTED: components
2026/09/18 13:16:35 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:components) ===
2026/09/18 13:16:36 === CLUSTER STATE ===
2026/09/18 13:16:36 Node: ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:16:36   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:16:36   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:16:36   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:16:36   Resources: 15500m CPU, 63871024Ki memory allocatable (of 16000m CPU, 65022000Ki memory capacity)
2026/09/18 13:16:36 Node: ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/18 13:16:36   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/18 13:16:36   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/18 13:16:36   Condition Ready: True - kubelet is posting ready status
2026/09/18 13:16:36   Resources: 15500m CPU, 63182884Ki memory allocatable (of 16000m CPU, 64333860Ki memory capacity)
2026/09/18 13:16:36 === DEPLOYMENTS ===
2026/09/18 13:16:36 Namespace: redhat-ods-applications
2026/09/18 13:16:36   Deployment agent-ops-ui: 1/1 (READY)
2026/09/18 13:16:36   Deployment ai-gateway-operator: 1/1 (READY)
2026/09/18 13:16:36   Deployment dashboard-operator: 2/2 (READY)
2026/09/18 13:16:36   Deployment dashboard-redirect: 2/2 (READY)
2026/09/18 13:16:36   Deployment gen-ai-ui: 1/1 (READY)
2026/09/18 13:16:36   Deployment kserve-controller-manager: 1/1 (READY)
2026/09/18 13:16:36   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/18 13:16:36   Deployment kubeflow-trainer-controller-manager: 1/1 (READY)
2026/09/18 13:16:36   Deployment llmisvc-controller-manager: 1/1 (READY)
2026/09/18 13:16:36   Deployment maas-ui: 1/1 (READY)
2026/09/18 13:16:36   Deployment model-serving-api: 1/1 (READY)
2026/09/18 13:16:36   Deployment odh-model-controller: 1/1 (READY)
2026/09/18 13:16:36   Deployment postgres-deployment: 1/1 (READY)
2026/09/18 13:16:36   Deployment rhods-dashboard: 2/2 (READY)
2026/09/18 13:16:36   Deployment spark-operator-controller: 1/1 (READY)
2026/09/18 13:16:36   Deployment spark-operator-webhook: 1/1 (READY)
2026/09/18 13:16:36 Namespace: kube-system
2026/09/18 13:16:36 === PODS ===
2026/09/18 13:16:36 Failed to collect pod data: redhat-ods-applications/ai-gateway-operator-57f8cd74c7-2gd6r: container copy-manifests Completed (exit 0)
2026/09/18 13:16:36 Continuing with partial pod data to collect logs from problematic containers
2026/09/18 13:16:36 Namespace: redhat-ods-applications
2026/09/18 13:16:36   Pod ai-gateway-operator-57f8cd74c7-2gd6r: Running
2026/09/18 13:16:36     Container copy-manifests: ready=true, terminated: Completed (exit 0)
2026/09/18 13:16:36 Namespace kube-system: no problematic pods found
2026/09/18 13:16:36 === OPERATOR STATUS ===
2026/09/18 13:16:36 Operator deployment: 3/3 ready
2026/09/18 13:16:36 === DSCI STATUS ===
2026/09/18 13:16:36 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringStackAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; ThanosQuerierAvailable=False; PersesAvailable=False; NodeMetricsEndpointAvailable=False
2026/09/18 13:16:36 === DSC STATUS ===
2026/09/18 13:16:36 DSC default-dsc:
2026/09/18 13:16:36   Ready: True - 
2026/09/18 13:16:36   ProvisioningSucceeded: True - 
2026/09/18 13:16:36   ComponentsReady: True - 
2026/09/18 13:16:36   ModulesReady: True - 
2026/09/18 13:16:36   AIGatewayReady: True - 
2026/09/18 13:16:36   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/18 13:16:36   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/18 13:16:36   DashboardReady: True - 
2026/09/18 13:16:36   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/18 13:16:36   KserveLLMInferenceServiceDependencies: True - 
2026/09/18 13:16:36   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/18 13:16:36   KserveReady: True - 
2026/09/18 13:16:36   KueueReady: False - Component ManagementState is set to Removed
2026/09/18 13:16:36   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/18 13:16:36   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/18 13:16:36   ModelRegistryReady: False - Component ManagementState is set to Removed
2026/09/18 13:16:36   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/18 13:16:36   OGXReady: False - Module ManagementState is set to Removed
2026/09/18 13:16:36   ProvisioningProgress: True - 
2026/09/18 13:16:36   RayReady: False - Component ManagementState is set to Removed
2026/09/18 13:16:36   SparkOperatorReady: True - 
2026/09/18 13:16:36   TrainerReady: True - 
2026/09/18 13:16:36   TrainingOperatorReady: False - Component ManagementState is set to Removed
2026/09/18 13:16:36   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/18 13:16:36   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/18 13:16:36 === RECENT EVENTS ===
2026/09/18 13:16:36   13:16:13 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6cc5fc4bb7-vtps2_e4465cdf-c320-40c4-92af-ca67d77b27b4 became leader
2026/09/18 13:16:36   13:16:12 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-qc2q8_03347f68-7fa3-44d9-b0bb-2205baf3fbe8 became leader
2026/09/18 13:16:36   13:16:11 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-86d7b88897-szf5z_6e1298df-b725-453d-a70c-197bae8f5cbd became leader
2026/09/18 13:16:36   13:15:55 INFO Pod/odh-model-controller-86d7b88897-szf5z: Created - Created container: manager
2026/09/18 13:16:36   13:15:55 INFO Pod/odh-model-controller-86d7b88897-szf5z: Started - Started container manager
2026/09/18 13:16:36   13:15:54 INFO Pod/model-serving-api-7b65949446-s52sc: AddedInterface - Add eth0 [10.129.45.205/23] from ovn-kubernetes
2026/09/18 13:16:36   13:15:54 INFO Pod/kserve-controller-manager-75b9f799cd-qc2q8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 1.024s (1.024s including waiting). Image size: 240120555 bytes.
2026/09/18 13:16:36   13:15:54 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-vtps2: Started - Started container manager
2026/09/18 13:16:36   13:15:54 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-vtps2: Created - Created container: manager
2026/09/18 13:16:36   13:15:54 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-vtps2: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 971ms (971ms including waiting). Image size: 238650146 bytes.
2026/09/18 13:16:36   13:15:54 INFO Pod/kserve-controller-manager-75b9f799cd-qc2q8: Created - Created container: manager
2026/09/18 13:16:36   13:15:54 INFO Pod/kserve-controller-manager-75b9f799cd-qc2q8: Started - Started container manager
2026/09/18 13:16:36   13:15:54 INFO Pod/odh-model-controller-86d7b88897-szf5z: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1.328s (1.328s including waiting). Image size: 238841358 bytes.
2026/09/18 13:16:36   13:15:54 INFO Pod/model-serving-api-7b65949446-s52sc: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:16:36   13:15:54 INFO Pod/model-serving-api-7b65949446-s52sc: Created - Created container: server
2026/09/18 13:16:36   13:15:54 INFO Pod/model-serving-api-7b65949446-s52sc: Started - Started container server
2026/09/18 13:16:36   13:15:53 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:16:36   13:15:53 INFO Pod/odh-model-controller-86d7b88897-szf5z: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-86d7b88897-szf5z to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:15:53 INFO Pod/model-serving-api-7b65949446-s52sc: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-s52sc to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:15:53 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-vtps2: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-vtps2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:15:53 INFO Pod/kserve-controller-manager-75b9f799cd-qc2q8: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-qc2q8 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:15:53 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-vtps2: AddedInterface - Add eth0 [10.129.45.204/23] from ovn-kubernetes
2026/09/18 13:16:36   13:15:53 INFO ReplicaSet/odh-model-controller-86d7b88897: SuccessfulCreate - Created pod: odh-model-controller-86d7b88897-szf5z
2026/09/18 13:16:36   13:15:53 WARN Pod/model-serving-api-7b65949446-s52sc: FailedMount - MountVolume.SetUp failed for volume "tls-certs" : secret "model-serving-api-tls" not found
2026/09/18 13:16:36   13:15:53 INFO Pod/kserve-controller-manager-75b9f799cd-qc2q8: AddedInterface - Add eth0 [10.129.45.203/23] from ovn-kubernetes
2026/09/18 13:16:36   13:15:53 INFO Pod/kserve-controller-manager-75b9f799cd-qc2q8: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:16:36   13:15:53 INFO Pod/odh-model-controller-86d7b88897-szf5z: AddedInterface - Add eth0 [10.128.15.230/23] from ovn-kubernetes
2026/09/18 13:16:36   13:15:53 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-s52sc
2026/09/18 13:16:36   13:15:53 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 0 to 1
2026/09/18 13:16:36   13:15:53 INFO Pod/odh-model-controller-86d7b88897-szf5z: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:16:36   13:15:53 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-qc2q8
2026/09/18 13:16:36   13:15:53 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-vtps2
2026/09/18 13:16:36   13:15:53 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-86d7b88897 from 0 to 1
2026/09/18 13:16:36   13:15:53 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-vtps2: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:16:36   13:15:52 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:16:36   13:15:52 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:16:36   13:15:27 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-6bb876bdfd-s89vq_fb05b19c-7f66-4f33-b2e8-a3397dc550a3 became leader
2026/09/18 13:16:36   13:15:07 INFO Pod/kserve-module-controller-manager-6bb876bdfd-s89vq: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-6bb876bdfd-s89vq to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:15:07 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-6bb876bdfd from 0 to 1
2026/09/18 13:16:36   13:15:07 INFO Pod/kserve-module-controller-manager-6bb876bdfd-s89vq: Created - Created container: manager
2026/09/18 13:16:36   13:15:07 INFO Pod/kserve-module-controller-manager-6bb876bdfd-s89vq: Started - Started container manager
2026/09/18 13:16:36   13:15:07 INFO Pod/kserve-module-controller-manager-6bb876bdfd-s89vq: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:c193f3d699bdd9209fcbe36bc42136687df03af8f43e3184809a56ee59e07568" already present on machine
2026/09/18 13:16:36   13:15:07 INFO Pod/kserve-module-controller-manager-6bb876bdfd-s89vq: AddedInterface - Add eth0 [10.128.15.229/23] from ovn-kubernetes
2026/09/18 13:16:36   13:15:07 INFO ReplicaSet/kserve-module-controller-manager-6bb876bdfd: SuccessfulCreate - Created pod: kserve-module-controller-manager-6bb876bdfd-s89vq
2026/09/18 13:16:36   13:14:58 INFO Pod/kserve-module-controller-manager-6bb876bdfd-vvc9w: Killing - Stopping container manager
2026/09/18 13:16:36   13:14:57 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: Killing - Stopping container manager
2026/09/18 13:16:36   13:14:57 INFO Pod/model-serving-api-7b65949446-xvpl7: Killing - Stopping container server
2026/09/18 13:16:36   13:14:57 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: Killing - Stopping container manager
2026/09/18 13:16:36   13:14:57 INFO Pod/odh-model-controller-86d7b88897-qbvlq: Killing - Stopping container manager
2026/09/18 13:16:36   13:14:34 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6cc5fc4bb7-h5pzb_fd373efd-15d0-4004-a525-2e942328bcdb became leader
2026/09/18 13:16:36   13:14:33 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-86d7b88897-qbvlq_10477db3-85e3-46ff-aa2f-bc27b93ae267 became leader
2026/09/18 13:16:36   13:14:30 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-75b9f799cd-lrwzh_5c1b9932-5b90-4e5f-831b-1c3d86d0ff0f became leader
2026/09/18 13:16:36   13:14:15 INFO Pod/odh-model-controller-86d7b88897-qbvlq: Created - Created container: manager
2026/09/18 13:16:36   13:14:15 INFO Pod/odh-model-controller-86d7b88897-qbvlq: Started - Started container manager
2026/09/18 13:16:36   13:14:14 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510" in 990ms (990ms including waiting). Image size: 240120555 bytes.
2026/09/18 13:16:36   13:14:14 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: Created - Created container: manager
2026/09/18 13:16:36   13:14:14 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: Started - Started container manager
2026/09/18 13:16:36   13:14:14 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: Created - Created container: manager
2026/09/18 13:16:36   13:14:14 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: Started - Started container manager
2026/09/18 13:16:36   13:14:14 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718" in 980ms (980ms including waiting). Image size: 238650146 bytes.
2026/09/18 13:16:36   13:14:14 INFO Pod/odh-model-controller-86d7b88897-qbvlq: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825" in 1s (1s including waiting). Image size: 238841358 bytes.
2026/09/18 13:16:36   13:14:13 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:16:36   13:14:13 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:8acbe4b18a3c3280aa268f4b20a4d8058b5f741fcd390f8c6c826dbd0f108510"
2026/09/18 13:16:36   13:14:13 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: AddedInterface - Add eth0 [10.129.45.201/23] from ovn-kubernetes
2026/09/18 13:16:36   13:14:13 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: AddedInterface - Add eth0 [10.129.45.200/23] from ovn-kubernetes
2026/09/18 13:16:36   13:14:13 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:d5cf593e040d9c370116dcf1b824774bc58e5303def3f1c5cd91788f13239718"
2026/09/18 13:16:36   13:14:13 INFO Pod/model-serving-api-7b65949446-xvpl7: AddedInterface - Add eth0 [10.129.45.202/23] from ovn-kubernetes
2026/09/18 13:16:36   13:14:13 INFO Pod/model-serving-api-7b65949446-xvpl7: Created - Created container: server
2026/09/18 13:16:36   13:14:13 INFO Pod/odh-model-controller-86d7b88897-qbvlq: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:f6b661f10314ad008f6baf42407ecb2e3a4cdab3366c6980173fb669ef17a825"
2026/09/18 13:16:36   13:14:13 INFO Pod/model-serving-api-7b65949446-xvpl7: Started - Started container server
2026/09/18 13:16:36   13:14:13 INFO Pod/model-serving-api-7b65949446-xvpl7: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:fe750574e29fc03f60fbdd48c54d6c8e48e8a0d96c357452369ff9e0a85a16a8" already present on machine
2026/09/18 13:16:36   13:14:13 INFO Pod/odh-model-controller-86d7b88897-qbvlq: AddedInterface - Add eth0 [10.128.15.228/23] from ovn-kubernetes
2026/09/18 13:16:36   13:14:13 WARN Pod/odh-model-controller-86d7b88897-qbvlq: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found
2026/09/18 13:16:36   13:14:12 INFO Pod/odh-model-controller-86d7b88897-qbvlq: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-86d7b88897-qbvlq to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:14:12 INFO Pod/model-serving-api-7b65949446-xvpl7: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7b65949446-xvpl7 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:14:12 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-h5pzb: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6cc5fc4bb7-h5pzb to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:14:12 INFO Pod/kserve-controller-manager-75b9f799cd-lrwzh: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-75b9f799cd-lrwzh to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:14:12 INFO ReplicaSet/llmisvc-controller-manager-6cc5fc4bb7: SuccessfulCreate - Created pod: llmisvc-controller-manager-6cc5fc4bb7-h5pzb
2026/09/18 13:16:36   13:14:12 INFO ReplicaSet/model-serving-api-7b65949446: SuccessfulCreate - Created pod: model-serving-api-7b65949446-xvpl7
2026/09/18 13:16:36   13:14:12 INFO ReplicaSet/odh-model-controller-86d7b88897: SuccessfulCreate - Created pod: odh-model-controller-86d7b88897-qbvlq
2026/09/18 13:16:36   13:14:12 INFO ReplicaSet/kserve-controller-manager-75b9f799cd: SuccessfulCreate - Created pod: kserve-controller-manager-75b9f799cd-lrwzh
2026/09/18 13:16:36   13:14:12 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-86d7b88897 from 0 to 1
2026/09/18 13:16:36   13:14:12 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-75b9f799cd from 0 to 1
2026/09/18 13:16:36   13:14:12 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6cc5fc4bb7 from 0 to 1
2026/09/18 13:16:36   13:14:12 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7b65949446 from 0 to 1
2026/09/18 13:16:36   13:13:48 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-6bb876bdfd-vvc9w_a30bb6ba-246a-41b8-9614-1806eb10a2bd became leader
2026/09/18 13:16:36   13:13:46 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7_18037c57-377f-43a4-977e-2d4f63574ccf became leader
2026/09/18 13:16:36   13:13:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7: Started - Started container manager
2026/09/18 13:16:36   13:13:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7: Created - Created container: manager
2026/09/18 13:16:36   13:13:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7: AddedInterface - Add eth0 [10.128.15.227/23] from ovn-kubernetes
2026/09/18 13:16:36   13:13:30 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:f529e1f052a39cdfc13f0352ce1660d24789cba772bcda85ee406307cdf3f515" already present on machine
2026/09/18 13:16:36   13:13:29 INFO Pod/kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:13:29 INFO ReplicaSet/kubeflow-trainer-controller-manager-65d74dc6fc: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-65d74dc6fc-lqbp7
2026/09/18 13:16:36   13:13:29 INFO Pod/rhods-dashboard-6b48d46697-vtsfs: Killing - Stopping container rhods-dashboard
2026/09/18 13:16:36   13:13:29 INFO Pod/rhods-dashboard-6b48d46697-vtsfs: Killing - Stopping container core-bff
2026/09/18 13:16:36   13:13:29 INFO Pod/rhods-dashboard-6b48d46697-vtsfs: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:16:36   13:13:29 INFO ReplicaSet/rhods-dashboard-6b48d46697: SuccessfulDelete - Deleted pod: rhods-dashboard-6b48d46697-vtsfs
2026/09/18 13:16:36   13:13:29 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-65d74dc6fc from 0 to 1
2026/09/18 13:16:36   13:13:27 INFO Pod/kserve-module-controller-manager-6bb876bdfd-vvc9w: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-6bb876bdfd-vvc9w to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:13:27 INFO Pod/kserve-module-controller-manager-6bb876bdfd-vvc9w: Created - Created container: manager
2026/09/18 13:16:36   13:13:27 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-6bb876bdfd from 0 to 1
2026/09/18 13:16:36   13:13:27 INFO Pod/kserve-module-controller-manager-6bb876bdfd-vvc9w: AddedInterface - Add eth0 [10.128.15.226/23] from ovn-kubernetes
2026/09/18 13:16:36   13:13:27 INFO Pod/kserve-module-controller-manager-6bb876bdfd-vvc9w: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:c193f3d699bdd9209fcbe36bc42136687df03af8f43e3184809a56ee59e07568" already present on machine
2026/09/18 13:16:36   13:13:27 INFO Pod/kserve-module-controller-manager-6bb876bdfd-vvc9w: Started - Started container manager
2026/09/18 13:16:36   13:13:27 INFO ReplicaSet/kserve-module-controller-manager-6bb876bdfd: SuccessfulCreate - Created pod: kserve-module-controller-manager-6bb876bdfd-vvc9w
2026/09/18 13:16:36   13:13:18 INFO Pod/maas-controller-f9dfb44dc-w2dvh: Killing - Stopping container manager
2026/09/18 13:16:36   13:13:03 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-f9dfb44dc-w2dvh_7bba66e4-9eb8-49f2-9a06-4861bafe1a58 became leader
2026/09/18 13:16:36   13:12:52 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 894ms (894ms including waiting). Image size: 179699994 bytes.
2026/09/18 13:16:36   13:12:52 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Created - Created container: core-bff
2026/09/18 13:16:36   13:12:52 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Started - Started container core-bff
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Started - Started container rhods-dashboard
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Started - Started container kube-rbac-proxy
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Created - Created container: kube-rbac-proxy
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Created - Created container: rhods-dashboard
2026/09/18 13:16:36   13:12:51 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 958ms (958ms including waiting). Image size: 824448782 bytes.
2026/09/18 13:16:36   13:12:50 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:16:36   13:12:50 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: AddedInterface - Add eth0 [10.129.45.199/23] from ovn-kubernetes
2026/09/18 13:16:36   13:12:49 INFO Pod/rhods-dashboard-76c555ffdd-mgck2: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-76c555ffdd-mgck2 to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:12:49 INFO ReplicaSet/rhods-dashboard-6b48d46697: SuccessfulDelete - Deleted pod: rhods-dashboard-6b48d46697-p5c4d
2026/09/18 13:16:36   13:12:49 INFO ReplicaSet/rhods-dashboard-76c555ffdd: SuccessfulCreate - Created pod: rhods-dashboard-76c555ffdd-mgck2
2026/09/18 13:16:36   13:12:49 INFO Deployment/rhods-dashboard: ScalingReplicaSet - (combined from similar events): Scaled down replica set rhods-dashboard-6b48d46697 from 2 to 1
2026/09/18 13:16:36   13:12:49 INFO Pod/rhods-dashboard-6b48d46697-p5c4d: Killing - Stopping container kube-rbac-proxy
2026/09/18 13:16:36   13:12:49 WARN Pod/rhods-dashboard-6b48d46697-p5c4d: Unhealthy - Readiness probe failed: Get "https://10.128.15.220:8444/healthz": dial tcp 10.128.15.220:8444: connect: connection refused
2026/09/18 13:16:36   13:12:49 INFO Pod/rhods-dashboard-6b48d46697-p5c4d: Killing - Stopping container rhods-dashboard
2026/09/18 13:16:36   13:12:49 INFO Pod/rhods-dashboard-6b48d46697-p5c4d: Killing - Stopping container core-bff
2026/09/18 13:16:36   13:12:45 INFO Pod/maas-controller-f9dfb44dc-w2dvh: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:31870540df9132209a2865ddc3aeff7cc04ea3fdcb044c2e3b3ee18dcb73a688" in 1.059s (1.059s including waiting). Image size: 183271983 bytes.
2026/09/18 13:16:36   13:12:45 INFO Pod/maas-controller-f9dfb44dc-w2dvh: Started - Started container manager
2026/09/18 13:16:36   13:12:45 INFO Pod/maas-controller-f9dfb44dc-w2dvh: Created - Created container: manager
2026/09/18 13:16:36   13:12:44 INFO Pod/maas-controller-f9dfb44dc-w2dvh: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:31870540df9132209a2865ddc3aeff7cc04ea3fdcb044c2e3b3ee18dcb73a688"
2026/09/18 13:16:36   13:12:44 INFO Pod/maas-controller-f9dfb44dc-w2dvh: AddedInterface - Add eth0 [10.129.45.198/23] from ovn-kubernetes
2026/09/18 13:16:36   13:12:43 INFO Pod/maas-controller-f9dfb44dc-w2dvh: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-f9dfb44dc-w2dvh to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:12:43 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-f9dfb44dc from 0 to 1
2026/09/18 13:16:36   13:12:43 INFO ReplicaSet/maas-controller-f9dfb44dc: SuccessfulCreate - Created pod: maas-controller-f9dfb44dc-w2dvh
2026/09/18 13:16:36   13:12:35 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-57f8cd74c7-2gd6r_735ae9a2-f977-4f03-a1f0-9b4fe702f2ec became leader
2026/09/18 13:16:36   13:12:18 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Created - Created container: manager
2026/09/18 13:16:36   13:12:18 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Started - Started container manager
2026/09/18 13:16:36   13:12:17 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:a30a011a8fb1f34adbc2c838d48ed71ff954a9668cb8b14e3ea920781c374257" in 1.135s (1.135s including waiting). Image size: 205332946 bytes.
2026/09/18 13:16:36   13:12:16 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:a30a011a8fb1f34adbc2c838d48ed71ff954a9668cb8b14e3ea920781c374257"
2026/09/18 13:16:36   13:12:15 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Started - Started container copy-manifests
2026/09/18 13:16:36   13:12:15 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:a30a011a8fb1f34adbc2c838d48ed71ff954a9668cb8b14e3ea920781c374257" in 1.145s (1.145s including waiting). Image size: 205332946 bytes.
2026/09/18 13:16:36   13:12:15 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Created - Created container: copy-manifests
2026/09/18 13:16:36   13:12:14 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: AddedInterface - Add eth0 [10.129.45.197/23] from ovn-kubernetes
2026/09/18 13:16:36   13:12:14 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:a30a011a8fb1f34adbc2c838d48ed71ff954a9668cb8b14e3ea920781c374257"
2026/09/18 13:16:36   13:12:13 INFO Pod/ai-gateway-operator-57f8cd74c7-2gd6r: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-57f8cd74c7-2gd6r to ip-10-0-1-213.us-east-2.compute.internal
2026/09/18 13:16:36   13:12:13 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-57f8cd74c7 from 0 to 1
2026/09/18 13:16:36   13:12:13 INFO ReplicaSet/ai-gateway-operator-57f8cd74c7: SuccessfulCreate - Created pod: ai-gateway-operator-57f8cd74c7-2gd6r
2026/09/18 13:16:36   13:12:12 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Started - Started container core-bff
2026/09/18 13:16:36   13:12:12 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed" in 1.051s (1.051s including waiting). Image size: 179699994 bytes.
2026/09/18 13:16:36   13:12:12 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Created - Created container: core-bff
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:179b20a2f7d4cb513b5735327197459505f97ac5a1e312fd9d535b7a76d56aed"
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Started - Started container kube-rbac-proxy
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Created - Created container: kube-rbac-proxy
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:b0ed5110949f527e26e6fb3505233f4b19377aca806dfacd2d9e6c7b927a719d" already present on machine
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Started - Started container rhods-dashboard
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Created - Created container: rhods-dashboard
2026/09/18 13:16:36   13:12:11 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5" in 1.27s (1.27s including waiting). Image size: 824448782 bytes.
2026/09/18 13:16:36   13:12:09 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-76c555ffdd-2tjq8 to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:12:09 INFO ReplicaSet/rhods-dashboard-76c555ffdd: SuccessfulCreate - Created pod: rhods-dashboard-76c555ffdd-2tjq8
2026/09/18 13:16:36   13:12:09 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: AddedInterface - Add eth0 [10.128.15.223/23] from ovn-kubernetes
2026/09/18 13:16:36   13:12:09 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-76c555ffdd from 0 to 1
2026/09/18 13:16:36   13:12:09 INFO Pod/rhods-dashboard-76c555ffdd-2tjq8: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:dac93d243ffefe5164c7f337613efdbbe390f9d242965a1475b8250fecf612f5"
2026/09/18 13:16:36   13:12:09 INFO Pod/eval-hub-ui-65bccdf794-hbrmx: Killing - Stopping container eval-hub-ui
2026/09/18 13:16:36   13:12:02 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:55 INFO Pod/kserve-module-controller-manager-6bb876bdfd-rh6tm: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:54 INFO Pod/odh-model-controller-86d7b88897-znwtw: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:54 INFO Pod/model-serving-api-7b65949446-qb9lh: Killing - Stopping container server
2026/09/18 13:16:36   13:11:54 INFO Pod/kserve-controller-manager-75b9f799cd-c7t4q: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:54 INFO Pod/llmisvc-controller-manager-6cc5fc4bb7-9t6j6: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:53 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/18 13:16:36   13:11:52 INFO Lease/b7e9931f.trustyai.opendatahub.io: LeaderElection - trustyai-service-operator-controller-manager-5d9d88f85f-9j97t_9695a33d-7d1d-4a7c-80db-a82707040055 became leader
2026/09/18 13:16:36   13:11:37 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t: Created - Created container: manager
2026/09/18 13:16:36   13:11:37 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t: Started - Started container manager
2026/09/18 13:16:36   13:11:36 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t: Scheduled - Successfully assigned redhat-ods-applications/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t to ip-10-0-1-93.us-east-2.compute.internal
2026/09/18 13:16:36   13:11:36 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t: AddedInterface - Add eth0 [10.128.15.222/23] from ovn-kubernetes
2026/09/18 13:16:36   13:11:36 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-9j97t: Pulled - Container image "quay.io/rhoai/odh-trustyai-service-operator-rhel9@sha256:73796f877b6914af47614cfe485559341b3dd74273cdd20e13dc9cfa0680596e" already present on machine
2026/09/18 13:16:36   13:11:36 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-c28pr: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:36 INFO Pod/trustyai-service-operator-controller-manager-5d9d88f85f-hvfwl: Killing - Stopping container manager
2026/09/18 13:16:36   13:11:36 INFO ReplicaSet/trustyai-service-operator-controller-manager-5d9d88f85f: SuccessfulCreate - Created pod: trustyai-service-operator-controller-manager-5d9d88f85f-9j97t
2026/09/18 13:16:36   13:11:36 INFO Deployment/trustyai-service-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trustyai-service-operator-controller-manager-5d9d88f85f from 0 to 1
2026/09/18 13:16:36 === RESOURCE QUOTAS ===
2026/09/18 13:16:36 No resource quotas found in monitored namespaces
2026/09/18 13:16:36 Classification for components: infrastructure/storage (code=1006, confidence=medium)
2026/09/18 13:16:36   Evidence: event Pod/model-serving-api-7b65949446-s52sc: FailedMount - MountVolume.SetUp failed for volume "tls-certs" : secret "model-serving-api-tls" not found
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"storage","error_code":1006,"evidence":["event Pod/model-serving-api-7b65949446-s52sc: FailedMount - MountVolume.SetUp failed for volume \"tls-certs\" : secret \"model-serving-api-tls\" not found"],"confidence":"medium"}
2026/09/18 13:16:36 Diagnostics complete for failed test: components
    controller_test.go:495: Stopping: components test failed.
2026/09/18 13:16:36 Circuit breaker: failure classified as infrastructure/storage (code=1006)
2026/09/18 13:16:36 Circuit breaker: infrastructure/unknown failure recorded (2/3)
    controller_test.go:767: Skipping tests for services/monitoring
Deletion Policy: Never. Skipping deletion tests.
2026/09/18 13:16:37 Circuit breaker: no trips during this run (threshold=3)
    controller_test.go:767: Skipping tests for services/monitoring]]></system-out>
	</testsuite>
</testsuites>
