<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="330" failures="1" skipped="93">
	<testsuite name="" tests="330" failures="1" errors="0" id="0" hostname="e2e-cli-nmanos-ephc-rhoai-smoke-vxgwz-test-platform-pod" skipped="93" time="7484.900" timestamp="2026-09-07T11:34:23Z">
		<testcase name="TestOdhOperator" classname="" time="1037.320">
			<system-out><![CDATA[2026/09/07 11:17:06 ARTIFACT_DIR not set, skipping periodic metrics collection
2026-09-07T11:17:06Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests" classname="" time="0.190">
			<system-out><![CDATA[2026-09-07T11:17:06Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Datascience_Cluster_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataScienceCluster_Initialization_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_FeatureTracker_CRD" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Ray_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrustyAI_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Kueue_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataSciencePipelines_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Auth_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Platform_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests" classname="" time="0.610">
			<system-out><![CDATA[2026-09-07T11:17:07Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_HardwareProfile_resource" classname="" time="10.140"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_owned_namespaces_exist" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_default_NetworkPolicy_exist" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_denies_invalid_PEM_on_update" classname="" time="0.090"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_valid_PEM_on_update" classname="" time="0.190"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_invalid_PEM_when_Removed" classname="" time="0.170"></testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests" classname="" time="11.130">
			<system-out><![CDATA[2026-09-07T11:17:17Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Ensuring clean slate: setting all components to Removed
    controller_test.go:762: Patched Subscription to remove env vars: [RHAI_VERSION CI]]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_in-tree_gates_block_and_unblock_provisioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_admin_ack_gates_block_and_unblock_provisioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_partial_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_runlevel_gating_and_convergence" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_PlatformReady_condition_on_component_CRs" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_component_stability_across_enable/disable_cycles_(RHOAIENG-73142)" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_DAG_cleanup" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/monitoring" classname="" time="0.140">
			<skipped message="Skipped"><![CDATA[2026-09-07T11:17:28Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components" classname="" time="1013.510"></testcase>
		<testcase name="TestOdhOperator/components/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway" classname="" time="70.780"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard" classname="" time="187.190">
			<system-out><![CDATA[2026-09-07T11:18:19Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines" classname="" time="86.340">
			<system-out><![CDATA[2026-09-07T11:18:30Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator" classname="" time="65.670">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve" classname="" time="433.790"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator" classname="" time="60.990"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx" classname="" time="71.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray" classname="" time="91.220">
			<system-out><![CDATA[2026-09-07T11:18:10Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator" classname="" time="141.360"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer" classname="" time="50.870"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches" classname="" time="40.980"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_enabled" classname="" time="30.510">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_enabled" classname="" time="101.360">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_enabled" classname="" time="60.420">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_enabled" classname="" time="40.510">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_enabled" classname="" time="50.670">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade_from_in-tree:_selector_migration" classname="" time="0.030">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_enabled" classname="" time="60.730">
			<system-out><![CDATA[2026-09-07T11:17:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_enabled" classname="" time="130.910"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_component_enabled" classname="" time="0.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_operator_deployed" classname="" time="25.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_created" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_ready" classname="" time="30.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_operator_deployment" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_workbenches_namespace_configuration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_releases" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_ImageStreams_available" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_MLflow_integration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_WorkbenchesV2_default_Removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No ConfigMap resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Service resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Deployment resources found for component Workbenches, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_disabled" classname="" time="10.240"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_module_handler_projects_DSC_config_to_Module_CR" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_disabled" classname="" time="10.180"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_WorkbenchesV2_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_enabled" classname="" time="30.440"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_disabled" classname="" time="10.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_component_enabled" classname="" time="40.690"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_feast-operator_deployed_by_module" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade:_existing_operands_preserved" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_disabled_cleanup" classname="" time="10.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_with_mixed_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_disabled" classname="" time="10.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_disabled" classname="" time="10.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_enabled" classname="" time="20.430"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_operands_have_OwnerReferences" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources/deployment_kuberay-operator" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_conditions" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources" classname="" time="20.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources/deployment_data-science-pipelines-operator-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery" classname="" time="30.330"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_ray-config-bf2bfgg6b4" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_operands_have_OwnerReferences" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_update_operand_resources" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="10.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-operator" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_spec" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_operands_have_OwnerReferences" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_no_Kserve_FeatureTrackers" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_VAP_created_when_kserve_is_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources" classname="" time="100.260"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_argoWorkflowsControllers_options" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery" classname="" time="35.410"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="15.210"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-config" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-webhook-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kuberay-operator" classname="" time="15.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_dynamically_watches_operands" classname="" time="10.180"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_workflow-controller-configmap" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_data-science-pipelines-operator-service" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-localmodel-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_CRDs_reinstated" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_VAP_blocks_dashboard_HardwareProfile_and_AcceleratorProfile_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery" classname="" time="116.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_data-science-pipelines-operator-controller-manager" classname="" time="15.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_operator_bootstrap_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_SparkPi_workload_execution" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_ScheduledSparkApplication_workload_execution" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_operator_Deployment_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_disabled" classname="" time="10.250"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="105.660"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_agent-ops-params" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_automl-params" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_llmisvc-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_autorag-params" classname="" time="95.060">
			<failure message="Failed"></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_model-serving-api" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_odh-model-controller" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_well-known_LLMInferenceServiceConfig_versioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_subscription_dependency_conditions" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_platform_config_ConfigMap" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:156: Verifying platform config ConfigMap odh-kserve-config exists in namespace redhat-ods-applications.
    helper_test.go:156: Non-XKS platform: verifying cert-manager CA keys are absent.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_ModelRegistry_state_propagation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery" classname="" time="221.500"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="20.260"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_inferenceservice-config" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-modelcache-parameters" classname="" time="5.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_odh-model-controller-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="45.570"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-service" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-webhook-server-service" classname="" time="5.060">
			<system-out><![CDATA[    test_context_test.go:1326: 
        Timed out after 90.004s.
        Resource ConfigMap redhat-ods-applications/autorag-params was not properly recreated with new identity
        The function passed to Eventually failed at /workspace/tests/e2e/test_context_test.go:1307 with:
        Expected ConfigMap redhat-ods-applications/autorag-params to be recreated
        Expected
            <*unstructured.Unstructured | 0x0>: nil
        not to be nil
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-controller-manager-service" classname="" time="5.060">
			<system-out><![CDATA[2026/09/07 11:21:26 === CLUSTER STATE ===
2026/09/07 11:21:26 Node: ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:21:26   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:21:26   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:21:26   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:21:26   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 11:21:26 Node: ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:21:26   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:21:26   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:21:26   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:21:26   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:21:26 Node: ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:21:26   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:21:26   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:21:26   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:21:26   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:21:26 === DEPLOYMENTS ===
2026/09/07 11:21:26 Namespace: redhat-ods-applications
2026/09/07 11:21:26   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 11:21:26   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 11:21:26   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment kserve-controller-manager: 2/2 (READY)
2026/09/07 11:21:26   Deployment kserve-localmodel-controller-manager: 2/2 (READY)
2026/09/07 11:21:26   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/07 11:21:26   Deployment llmisvc-controller-manager: 2/2 (READY)
2026/09/07 11:21:26   Deployment maas-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment model-serving-api: 2/2 (READY)
2026/09/07 11:21:26   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment odh-model-controller: 2/2 (READY)
2026/09/07 11:21:26   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 11:21:26 Namespace: kube-system
2026/09/07 11:21:26 === PODS ===
2026/09/07 11:21:26 Namespace redhat-ods-applications: no problematic pods found
2026/09/07 11:21:26 Namespace kube-system: no problematic pods found
2026/09/07 11:21:26 === OPERATOR STATUS ===
2026/09/07 11:21:26 Operator deployment: 3/3 ready
2026/09/07 11:21:26 === DSCI STATUS ===
2026/09/07 11:21:26 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 11:21:26 === DSC STATUS ===
2026/09/07 11:21:26 DSC default-dsc:
2026/09/07 11:21:26   Ready: True - 
2026/09/07 11:21:26   ProvisioningSucceeded: True - 
2026/09/07 11:21:26   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 11:21:26   ModulesReady: True - 
2026/09/07 11:21:26   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:21:26   DashboardReady: True - 
2026/09/07 11:21:26   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   KserveLLMInferenceServiceDependencies: True - 
2026/09/07 11:21:26   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/07 11:21:26   KserveReady: True - 
2026/09/07 11:21:26   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:21:26   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   RayReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   TrainerReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 11:21:26 === RECENT EVENTS ===
2026/09/07 11:21:26   11:21:23 WARN Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
2026/09/07 11:21:26   11:21:16 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:16 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulDelete - Deleted pod: rhods-dashboard-7f94fcf99d-6cjmj
2026/09/07 11:21:26   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:21:16 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7f94fcf99d from 1 to 0
2026/09/07 11:21:26   11:21:11 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:11 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:05 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe failed: Get "https://10.133.0.111:8444/healthz": dial tcp 10.133.0.111:8444: connect: connection refused
2026/09/07 11:21:26   11:20:55 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:20:55 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe failed: Get "https://10.133.0.111:8943/healthcheck": dial tcp 10.133.0.111:8943: connect: connection refused
2026/09/07 11:21:26   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 388ms (388ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: core-bff
2026/09/07 11:21:26   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container core-bff
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 594ms (594ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-775867476c-f7trw: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-f7trw to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:20:36 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-775867476c from 1 to 2
2026/09/07 11:21:26   11:20:36 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulDelete - Deleted pod: rhods-dashboard-5c9d44b7b4-dgpl5
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:20:36 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-5c9d44b7b4 from 1 to 0
2026/09/07 11:21:26   11:20:36 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-f7trw
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 2.22s (2.22s including waiting). Image size: 239132625 bytes.
2026/09/07 11:21:26   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Started - Started container manager
2026/09/07 11:21:26   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Created - Created container: manager
2026/09/07 11:21:26   11:20:31 INFO Pod/odh-model-controller-57457c4c89-bgwgw: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/07 11:21:26   11:20:31 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:21:26   11:20:30 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-bgwgw to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:20:30 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/07 11:21:26   11:20:30 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-bgwgw
2026/09/07 11:21:26   11:20:27 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe failed: Get "https://10.133.0.114:8943/healthcheck": dial tcp 10.133.0.114:8943: connect: connection refused
2026/09/07 11:21:26   11:20:27 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:20:23 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe failed: Get "https://10.133.0.114:8444/healthz": dial tcp 10.133.0.114:8444: connect: connection refused
2026/09/07 11:21:26   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Created - Created container: server
2026/09/07 11:21:26   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.284s (1.284s including waiting). Image size: 172459788 bytes.
2026/09/07 11:21:26   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Started - Started container server
2026/09/07 11:21:26   11:20:11 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 11:21:26   11:20:11 INFO Pod/model-serving-api-7df8c9655f-dz7bh: AddedInterface - Add eth0 [10.134.0.112/23] from ovn-kubernetes
2026/09/07 11:21:26   11:20:10 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-dz7bh to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:20:10 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-dz7bh
2026/09/07 11:21:26   11:20:10 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/07 11:21:26   11:20:01 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe failed: Get "https://10.134.0.106:8444/healthz": dial tcp 10.134.0.106:8444: connect: connection refused
2026/09/07 11:21:26   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container core-bff
2026/09/07 11:21:26   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: core-bff
2026/09/07 11:21:26   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 373ms (373ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: AddedInterface - Add eth0 [10.134.0.111/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 429ms (429ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-775867476c-bpspj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-bpspj to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:57 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulDelete - Deleted pod: rhods-dashboard-7f94fcf99d-rwzbj
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:19:57 INFO Pod/autorag-ui-58f548cbd-8pn4g: Killing - Stopping container autorag-ui
2026/09/07 11:21:26   11:19:57 WARN Endpoints/odh-dashboard-automl-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-automl-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-automl-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-automl-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 9931d5c4-97af-4cd3-9b95-ee5288a56884, UID in object meta: 
2026/09/07 11:21:26   11:19:57 WARN Endpoints/odh-dashboard-autorag-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-autorag-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-autorag-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-autorag-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 79bb98aa-3dac-4c9d-ae1e-46c4f3573e85, UID in object meta: 
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:19:57 INFO Pod/automl-ui-74d9b97d49-lzkxj: Killing - Stopping container automl-ui
2026/09/07 11:21:26   11:19:57 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-775867476c from 0 to 1
2026/09/07 11:21:26   11:19:57 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7f94fcf99d from 2 to 1
2026/09/07 11:21:26   11:19:57 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-bpspj
2026/09/07 11:21:26   11:19:56 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe failed: Get "https://10.134.0.106:8943/healthcheck": dial tcp 10.134.0.106:8943: connect: connection refused
2026/09/07 11:21:26   11:19:56 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Created - Created container: manager
2026/09/07 11:21:26   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 1.941s (1.941s including waiting). Image size: 235279660 bytes.
2026/09/07 11:21:26   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Started - Started container manager
2026/09/07 11:21:26   11:19:51 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:21:26   11:19:51 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: AddedInterface - Add eth0 [10.132.0.104/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:50 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-hvhs7 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:19:50 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-hvhs7
2026/09/07 11:21:26   11:19:50 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/07 11:21:26   11:19:46 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:19:46 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:19:46 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:41 WARN Pod/spark-operator-webhook-5d466b7d66-g8r7p: Unhealthy - Readiness probe failed: Get "http://10.133.0.109:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/07 11:21:26   11:19:41 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:40 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Killing - Stopping container controller
2026/09/07 11:21:26   11:19:40 WARN Pod/spark-operator-controller-c594b967f-wpdtn: Unhealthy - Readiness probe failed: Get "http://10.133.0.108:8081/readyz": dial tcp 10.133.0.108:8081: connect: connection refused
2026/09/07 11:21:26   11:19:40 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Killing - Stopping container webhook
2026/09/07 11:21:26   11:19:36 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Created - Created container: core-bff
2026/09/07 11:21:26   11:19:36 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Started - Started container core-bff
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 391ms (391ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 579ms (579ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7f94fcf99d-rwzbj to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:19:34 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulDelete - Deleted pod: rhods-dashboard-5c9d44b7b4-zhmtt
2026/09/07 11:21:26   11:19:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-5c9d44b7b4 from 2 to 1
2026/09/07 11:21:26   11:19:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7f94fcf99d from 1 to 2
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: AddedInterface - Add eth0 [10.133.0.114/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:19:34 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulCreate - Created pod: rhods-dashboard-7f94fcf99d-rwzbj
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:19:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Created - Created container: manager
2026/09/07 11:21:26   11:19:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Started - Started container manager
2026/09/07 11:21:26   11:19:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Started - Started container manager
2026/09/07 11:21:26   11:19:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 2.448s (2.448s including waiting). Image size: 244249302 bytes.
2026/09/07 11:21:26   11:19:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.358s (1.358s including waiting). Image size: 178395693 bytes.
2026/09/07 11:21:26   11:19:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Created - Created container: manager
2026/09/07 11:21:26   11:19:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f897kgkm to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:31 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: AddedInterface - Add eth0 [10.132.0.102/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:31 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:19:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 11:21:26   11:19:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: AddedInterface - Add eth0 [10.134.0.110/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:31 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f897kgkm
2026/09/07 11:21:26   11:19:31 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:21:26   11:19:30 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-4lrwg to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:19:30 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:30 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 11:21:26   11:19:30 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-4lrwg
2026/09/07 11:21:26   11:19:30 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 1 to 2
2026/09/07 11:21:26   11:19:30 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-gzzrx to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: AddedInterface - Add eth0 [10.134.0.109/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Created - Created container: kuberay-operator
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Started - Started container kuberay-operator
2026/09/07 11:21:26   11:19:16 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-gzzrx
2026/09/07 11:21:26   11:19:16 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 11:21:26   11:19:15 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:19:15 INFO Pod/kuberay-operator-7d87f54798-z4s96: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:19:13 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Started - Started container manager
2026/09/07 11:21:26   11:19:13 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Created - Created container: manager
2026/09/07 11:21:26   11:19:13 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 2.189s (2.189s including waiting). Image size: 247667445 bytes.
2026/09/07 11:21:26   11:19:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:21:26   11:19:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: AddedInterface - Add eth0 [10.133.0.113/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:10 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-v76tn to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:19:10 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-v76tn
2026/09/07 11:21:26   11:19:10 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 1 to 2
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: Started - Started container manager
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" already present on machine
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: Created - Created container: manager
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: AddedInterface - Add eth0 [10.134.0.108/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:00 INFO Pod/dashboard-operator-669d6564bd-gcllm: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-gcllm to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:00 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-gcllm
2026/09/07 11:21:26   11:19:00 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 1 to 2
2026/09/07 11:21:26   11:18:56 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Created - Created container: core-bff
2026/09/07 11:21:26   11:18:56 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 364ms (364ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:18:56 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Started - Started container core-bff
2026/09/07 11:21:26   11:18:55 INFO Pod/automl-ui-74d9b97d49-lzkxj: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af" in 3.59s (3.59s including waiting). Image size: 222806700 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/automl-ui-74d9b97d49-lzkxj: Started - Started container automl-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:18:55 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3" in 3.473s (3.473s including waiting). Image size: 184646446 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 2.311s (2.311s including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:18:55 INFO Pod/notebooks-ui-6577cc467-6fxnb: Started - Started container notebooks-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Started - Started container agent-ops-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:18:55 INFO Pod/notebooks-ui-6577cc467-6fxnb: Created - Created container: notebooks-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/notebooks-ui-6577cc467-6fxnb: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003" in 3.13s (3.13s including waiting). Image size: 190593254 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/autorag-ui-58f548cbd-8pn4g: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2" in 3.257s (3.257s including waiting). Image size: 223947455 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:18:55 INFO Pod/automl-ui-74d9b97d49-lzkxj: Created - Created container: automl-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/autorag-ui-58f548cbd-8pn4g: Created - Created container: autorag-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Created - Created container: agent-ops-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/autorag-ui-58f548cbd-8pn4g: Started - Started container autorag-ui
2026/09/07 11:21:26   11:18:53 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:18:52 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7f94fcf99d-6cjmj to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:52 INFO Pod/notebooks-ui-6577cc467-6fxnb: AddedInterface - Add eth0 [10.132.0.100/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:52 INFO Pod/notebooks-ui-6577cc467-6fxnb: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003"
2026/09/07 11:21:26   11:18:52 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7f94fcf99d from 0 to 1
2026/09/07 11:21:26   11:18:52 INFO Pod/autorag-ui-58f548cbd-8pn4g: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2"
2026/09/07 11:21:26   11:18:52 INFO Pod/autorag-ui-58f548cbd-8pn4g: AddedInterface - Add eth0 [10.132.0.98/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:52 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulCreate - Created pod: rhods-dashboard-7f94fcf99d-6cjmj
2026/09/07 11:21:26   11:18:52 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: AddedInterface - Add eth0 [10.132.0.101/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/notebooks-ui-6577cc467-6fxnb: Scheduled - Successfully assigned redhat-ods-applications/notebooks-ui-6577cc467-6fxnb to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:51 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-7df989bc55-wh2xr to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:51 INFO Pod/maas-ui-675f94cfd8-57bsm: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe" in 442ms (442ms including waiting). Image size: 189581000 bytes.
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/07 11:21:26   11:18:51 WARN Pod/autorag-ui-58f548cbd-8pn4g: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "autorag-proxy-tls" not found
2026/09/07 11:21:26   11:18:51 INFO ReplicaSet/notebooks-ui-6577cc467: SuccessfulCreate - Created pod: notebooks-ui-6577cc467-6fxnb
2026/09/07 11:21:26   11:18:51 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-8j6h9_e619c468-c35f-48a9-8f85-59edaf9e7466 became leader
2026/09/07 11:21:26   11:18:51 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-cszzw_b4119a50-069e-4b0b-8e87-25a733e64a6a became leader
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: AddedInterface - Add eth0 [10.133.0.112/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/maas-ui-675f94cfd8-57bsm: Created - Created container: maas-ui
2026/09/07 11:21:26   11:18:51 INFO Deployment/notebooks-ui: ScalingReplicaSet - Scaled up replica set notebooks-ui-6577cc467 from 0 to 1
2026/09/07 11:21:26   11:18:51 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Started - Started container gen-ai-ui
2026/09/07 11:21:26   11:18:51 INFO Pod/automl-ui-74d9b97d49-lzkxj: AddedInterface - Add eth0 [10.132.0.97/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Created - Created container: manager
2026/09/07 11:21:26   11:18:51 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-7df989bc55 from 0 to 1
2026/09/07 11:21:26   11:18:51 INFO Pod/maas-ui-675f94cfd8-57bsm: Started - Started container maas-ui
2026/09/07 11:21:26   11:18:51 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: AddedInterface - Add eth0 [10.132.0.99/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Started - Started container manager
2026/09/07 11:21:26   11:18:51 INFO Pod/automl-ui-74d9b97d49-lzkxj: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af"
2026/09/07 11:21:26   11:18:51 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3"
2026/09/07 11:21:26   11:18:51 INFO ReplicaSet/agent-ops-ui-7df989bc55: SuccessfulCreate - Created pod: agent-ops-ui-7df989bc55-wh2xr
2026/09/07 11:21:26   11:18:50 INFO Pod/autorag-ui-58f548cbd-8pn4g: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-58f548cbd-8pn4g to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:50 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89f5np4 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:50 INFO Pod/automl-ui-74d9b97d49-lzkxj: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-74d9b97d49-lzkxj to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:50 INFO Pod/maas-ui-675f94cfd8-57bsm: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-675f94cfd8-57bsm to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:50 WARN Pod/automl-ui-74d9b97d49-lzkxj: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/07 11:21:26   11:18:50 INFO Pod/maas-ui-675f94cfd8-57bsm: AddedInterface - Add eth0 [10.132.0.96/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:50 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-wrr9b_fbc23ba2-8d7b-45c8-9965-936c02dedcd8 became leader
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89f5np4
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/autorag-ui-58f548cbd: SuccessfulCreate - Created pod: autorag-ui-58f548cbd-8pn4g
2026/09/07 11:21:26   11:18:50 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-58f548cbd from 0 to 1
2026/09/07 11:21:26   11:18:50 INFO Pod/maas-ui-675f94cfd8-57bsm: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe"
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/maas-ui-675f94cfd8: SuccessfulCreate - Created pod: maas-ui-675f94cfd8-57bsm
2026/09/07 11:21:26   11:18:50 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 1 to 2
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: AddedInterface - Add eth0 [10.134.0.107/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:50 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-74d9b97d49 from 0 to 1
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/automl-ui-74d9b97d49: SuccessfulCreate - Created pod: automl-ui-74d9b97d49-lzkxj
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987"
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987" in 477ms (477ms including waiting). Image size: 238860397 bytes.
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Created - Created container: gen-ai-ui
2026/09/07 11:21:26   11:18:49 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-6459ffd979-lqcdt to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:49 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-6459ffd979 from 0 to 1
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Started - Started container core-bff
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Started - Started container core-bff
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Created - Created container: core-bff
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 435ms (435ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:18:49 INFO ReplicaSet/gen-ai-ui-6459ffd979: SuccessfulCreate - Created pod: gen-ai-ui-6459ffd979-lqcdt
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 421ms (421ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:18:49 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-675f94cfd8 from 0 to 1
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Created - Created container: core-bff
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: AddedInterface - Add eth0 [10.134.0.106/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 467ms (467ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 596ms (597ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:18:47 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-5c9d44b7b4-zhmtt to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:47 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-5c9d44b7b4-dgpl5 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:47 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulCreate - Created pod: rhods-dashboard-5c9d44b7b4-zhmtt
2026/09/07 11:21:26   11:18:47 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulCreate - Created pod: rhods-dashboard-5c9d44b7b4-dgpl5
2026/09/07 11:21:26   11:18:47 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/07 11:21:26   11:18:47 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: AddedInterface - Add eth0 [10.133.0.111/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:47 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-5c9d44b7b4 from 0 to 2
2026/09/07 11:21:26   11:18:43 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7d87f54798-z4s96_bbd1a190-4af4-4ea7-a6dc-91a18c29f0fd became leader
2026/09/07 11:21:26   11:18:43 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-669d6564bd-twg9s_17baf0a1-3f70-4589-8a23-c2e37a314d8a became leader
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: AddedInterface - Add eth0 [10.134.0.105/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Created - Created container: kuberay-operator
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Started - Started container kuberay-operator
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 11:21:26   11:18:40 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-6pzmm to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:40 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 1 to 2
2026/09/07 11:21:26   11:18:40 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-6pzmm
2026/09/07 11:21:26   11:18:35 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Started - Started container server
2026/09/07 11:21:26   11:18:35 INFO Pod/kserve-localmodelnode-agent-4psfs: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 1.09s (1.09s including waiting). Image size: 244000959 bytes.
2026/09/07 11:21:26   11:18:35 INFO Pod/kserve-localmodelnode-agent-4psfs: Created - Created container: manager
2026/09/07 11:21:26   11:18:35 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Created - Created container: server
2026/09/07 11:21:26   11:18:35 INFO Pod/kserve-localmodelnode-agent-4psfs: Started - Started container manager
2026/09/07 11:21:26   11:18:35 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Started - Started container manager
2026/09/07 11:21:26   11:18:35 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Created - Created container: manager
2026/09/07 11:21:26   11:18:35 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.706s (1.706s including waiting). Image size: 178395693 bytes.
2026/09/07 11:21:26   11:18:35 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 2.128s (2.128s including waiting). Image size: 172459788 bytes.
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Created - Created container: manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-4psfs: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:21:26   11:18:34 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Started - Started container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-c65mt: Started - Started container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 505ms (505ms including waiting). Image size: 247667445 bytes.
2026/09/07 11:21:26   11:18:34 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Created - Created container: manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-4psfs: AddedInterface - Add eth0 [10.132.0.95/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-c65mt: Created - Created container: manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-c65mt: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 720ms (720ms including waiting). Image size: 244000959 bytes.
2026/09/07 11:21:26   11:18:34 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Started - Started container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 527ms (527ms including waiting). Image size: 239132625 bytes.
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-c65mt: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-c65mt to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-4psfs: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-4psfs to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Started - Started container manager
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 484ms (484ms including waiting). Image size: 244249302 bytes.
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-c65mt: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:21:26   11:18:33 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-4psfs
2026/09/07 11:21:26   11:18:33 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:21:26   11:18:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 593ms (593ms including waiting). Image size: 235279660 bytes.
2026/09/07 11:21:26   11:18:33 INFO Pod/odh-model-controller-57457c4c89-8j6h9: AddedInterface - Add eth0 [10.132.0.93/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Created - Created container: manager
2026/09/07 11:21:26   11:18:33 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:21:26   11:18:33 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: AddedInterface - Add eth0 [10.134.0.102/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:33 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-c65mt
2026/09/07 11:21:26   11:18:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: AddedInterface - Add eth0 [10.132.0.94/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 11:21:26   11:18:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Created - Created container: manager
2026/09/07 11:21:26   11:18:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Started - Started container manager
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-c65mt: AddedInterface - Add eth0 [10.134.0.104/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89fwzvr to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:32 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:32 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:32 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-8j6h9 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-v8wbk to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-cszzw to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-rgvs7 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-wrr9b to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: AddedInterface - Add eth0 [10.134.0.103/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-8j6h9
2026/09/07 11:21:26   11:18:32 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 11:21:26   11:18:32 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "run-as-ray-user": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": For
bidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-wrr9b
2026/09/07 11:21:26   11:18:32 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89fwzvr
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-rgvs7
2026/09/07 11:21:26   11:18:32 INFO Pod/model-serving-api-7df8c9655f-v8wbk: AddedInterface - Add eth0 [10.132.0.92/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: AddedInterface - Add eth0 [10.133.0.110/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 WARN Pod/odh-model-controller-57457c4c89-8j6h9: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found
2026/09/07 11:21:26   11:18:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:21:26   11:18:32 WARN Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kserve-webhook-server-cert" not found
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-v8wbk
2026/09/07 11:21:26   11:18:32 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-cszzw
2026/09/07 11:21:26   11:18:32 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "run-as-ray-user": Forbidden: not usable by user or serviceaccount, provider "openshift-ai-inferenceservice-image-volume-scc": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceac
count, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:21:26   11:18:30 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:29 INFO Pod/maas-controller-56c898799-lnmx8: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:26 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:26 INFO Lease/spark-operator-controller-lock: LeaderElection - spark-operator-controller-c594b967f-wpdtn_4159f6ac-8b27-4ac6-b4df-23a5801dff94 became leader
2026/09/07 11:21:26   11:18:25 INFO Pod/dashboard-operator-669d6564bd-twg9s: Started - Started container manager
2026/09/07 11:21:26   11:18:25 INFO Pod/dashboard-operator-669d6564bd-twg9s: Created - Created container: manager
2026/09/07 11:21:26   11:18:25 INFO Pod/dashboard-operator-669d6564bd-twg9s: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" in 1.431s (1.431s including waiting). Image size: 170934264 bytes.
2026/09/07 11:21:26   11:18:24 INFO Pod/dashboard-operator-669d6564bd-twg9s: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520"
2026/09/07 11:21:26   11:18:24 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-5d466b7d66-g8r7p_cc4b67b7-4769-455e-8ae2-4fa8a45217e9 became leader
2026/09/07 11:21:26   11:18:24 INFO Pod/dashboard-operator-669d6564bd-twg9s: AddedInterface - Add eth0 [10.132.0.91/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:24 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:24 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-59777cdbbb-stntf_4d39fb2b-23e8-4a3b-8da3-465cf3cd2dd3 became leader
2026/09/07 11:21:26   11:18:23 INFO Pod/dashboard-operator-669d6564bd-twg9s: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-twg9s to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:23 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-twg9s
2026/09/07 11:21:26   11:18:23 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 0 to 1
2026/09/07 11:21:26   11:18:22 INFO Pod/kuberay-operator-7d87f54798-z4s96: Started - Started container kuberay-operator
2026/09/07 11:21:26   11:18:22 INFO Pod/kuberay-operator-7d87f54798-z4s96: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.937s (1.937s including waiting). Image size: 189730715 bytes.
2026/09/07 11:21:26   11:18:22 INFO Pod/kuberay-operator-7d87f54798-z4s96: Created - Created container: kuberay-operator
2026/09/07 11:21:26   11:18:22 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: AddedInterface - Add eth0 [10.134.0.101/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Started - Started container nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Created - Created container: nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Created - Created container: nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Started - Started container nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: AddedInterface - Add eth0 [10.132.0.90/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 11:21:26   11:18:20 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-55f4444684-zpwrc to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:20 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-55f4444684-bs9jf to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:20 INFO Pod/kuberay-operator-7d87f54798-z4s96: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/07 11:21:26   11:18:20 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-55f4444684 from 0 to 2
2026/09/07 11:21:26   11:18:20 INFO Pod/kuberay-operator-7d87f54798-z4s96: AddedInterface - Add eth0 [10.132.0.89/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:20 INFO ReplicaSet/dashboard-redirect-55f4444684: SuccessfulCreate - Created pod: dashboard-redirect-55f4444684-bs9jf
2026/09/07 11:21:26   11:18:20 INFO ReplicaSet/dashboard-redirect-55f4444684: SuccessfulCreate - Created pod: dashboard-redirect-55f4444684-zpwrc
2026/09/07 11:21:26   11:18:19 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:19 INFO Pod/kuberay-operator-7d87f54798-z4s96: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-z4s96 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:19 WARN Pod/kuberay-operator-7d87f54798-z4s96: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/07 11:21:26   11:18:19 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:19 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-z4s96
2026/09/07 11:21:26   11:18:19 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 11:21:26   11:18:17 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:17 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:16 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:16 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:15 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:11 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Started - Started container manager
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-5fc6bb8d67-c4sfm to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Created - Created container: manager
2026/09/07 11:21:26   11:18:10 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-5fc6bb8d67 from 0 to 1
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: AddedInterface - Add eth0 [10.132.0.88/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:10 INFO ReplicaSet/feast-operator-controller-manager-5fc6bb8d67: SuccessfulCreate - Created pod: feast-operator-controller-manager-5fc6bb8d67-c4sfm
2026/09/07 11:21:26   11:18:10 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Pulled - Container image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7" already present on machine
2026/09/07 11:21:26   11:18:10 INFO Pod/maas-controller-56c898799-lnmx8: Started - Started container manager
2026/09/07 11:21:26   11:18:10 INFO Pod/maas-controller-56c898799-lnmx8: Created - Created container: manager
2026/09/07 11:21:26   11:18:10 INFO Pod/maas-controller-56c898799-lnmx8: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd" in 2.095s (2.095s including waiting). Image size: 197531122 bytes.
2026/09/07 11:21:26   11:18:09 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Created - Created container: manager
2026/09/07 11:21:26   11:18:09 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2" in 1.707s (1.707s including waiting). Image size: 193888070 bytes.
2026/09/07 11:21:26   11:18:09 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Started - Started container manager
2026/09/07 11:21:26   11:18:08 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:08 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-webhook-5d466b7d66-g8r7p to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-controller-c594b967f-wpdtn to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Created - Created container: webhook
2026/09/07 11:21:26   11:18:08 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Started - Started container webhook
2026/09/07 11:21:26   11:18:08 INFO ReplicaSet/spark-operator-controller-c594b967f: SuccessfulCreate - Created pod: spark-operator-controller-c594b967f-wpdtn
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Started - Started container controller
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 11:21:26   11:18:08 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:08 INFO ReplicaSet/spark-operator-webhook-5d466b7d66: SuccessfulCreate - Created pod: spark-operator-webhook-5d466b7d66-g8r7p
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: AddedInterface - Add eth0 [10.133.0.108/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: AddedInterface - Add eth0 [10.133.0.109/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:08 INFO Pod/maas-controller-56c898799-lnmx8: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd"
2026/09/07 11:21:26   11:18:08 INFO Pod/maas-controller-56c898799-lnmx8: AddedInterface - Add eth0 [10.132.0.87/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Created - Created container: controller
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 11:21:26   11:18:07 INFO Pod/maas-controller-56c898799-lnmx8: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-56c898799-lnmx8 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:07 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-59777cdbbb-stntf to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:07 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-59777cdbbb from 0 to 1
2026/09/07 11:21:26   11:18:07 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: AddedInterface - Add eth0 [10.132.0.86/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:07 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/07 11:21:26   11:18:07 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-56c898799 from 0 to 1
2026/09/07 11:21:26   11:18:07 INFO ReplicaSet/ogx-k8s-operator-controller-manager-59777cdbbb: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-59777cdbbb-stntf
2026/09/07 11:21:26   11:18:07 INFO ReplicaSet/maas-controller-56c898799: SuccessfulCreate - Created pod: maas-controller-56c898799-lnmx8
2026/09/07 11:21:26   11:18:07 INFO Deployment/spark-operator-controller: ScalingReplicaSet - Scaled up replica set spark-operator-controller-c594b967f from 0 to 1
2026/09/07 11:21:26   11:18:07 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2"
2026/09/07 11:21:26   11:18:05 INFO Pod/workbenches-operator-c674d9c9-p42hk: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:04 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Created - Created container: manager
2026/09/07 11:21:26   11:18:04 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5_757d5306-0f25-47ed-8a65-f3953293ce91 became leader
2026/09/07 11:21:26   11:18:04 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Started - Started container manager
2026/09/07 11:21:26   11:18:04 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728" in 2.346s (2.346s including waiting). Image size: 186574763 bytes.
2026/09/07 11:21:26   11:18:02 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-84glx_683221f7-bd93-4938-8e21-01ee7cd15381 became leader
2026/09/07 11:21:26   11:18:01 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:01 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:01 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:01 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-5f798b84bf from 0 to 1
2026/09/07 11:21:26   11:18:01 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728"
2026/09/07 11:21:26   11:18:01 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:01 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:01 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-5f798b84bf: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5
2026/09/07 11:21:26   11:18:01 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: AddedInterface - Add eth0 [10.133.0.107/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:00 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-644d45d5b-bvlbv_807ffc24-612b-4923-8d3d-45464ff69e83 became leader
2026/09/07 11:21:26   11:17:59 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-dqh6n_b9bab43a-f849-4207-81cf-484344a0a195 became leader
2026/09/07 11:21:26   11:17:59 INFO Lease/opendatahub-feast-operator-lock: LeaderElection - opendatahub-feast-operator-84ff599545-lnwt6_77e2eb9d-3227-4929-b86a-065587d1ed38 became leader
2026/09/07 11:21:26   11:17:59 INFO Lease/spark-operator-module-controller-manager: LeaderElection - spark-operator-module-controller-manager-59549b5854-sclh9_81261d89-5699-4c77-8e2c-5fcb011b0d28 became leader
2026/09/07 11:21:26   11:17:58 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-89f69845b-lqv8q_2720f88b-2549-477e-a17f-7cd0e599056f became leader
2026/09/07 11:21:26   11:17:49 WARN Pod/rhods-dashboard-bfcc58b46-v2xm8: Unhealthy - Readiness probe failed: Get "https://10.132.0.55:8444/healthz": dial tcp 10.132.0.55:8444: connect: connection refused
2026/09/07 11:21:26   11:17:49 WARN Pod/rhods-dashboard-bfcc58b46-42vcs: Unhealthy - Readiness probe failed: Get "https://10.133.0.52:8444/healthz": dial tcp 10.133.0.52:8444: connect: connection refused
2026/09/07 11:21:26   11:17:43 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Started - Started container manager
2026/09/07 11:21:26   11:17:43 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Started - Started container manager
2026/09/07 11:21:26   11:17:42 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Started - Started container manager
2026/09/07 11:21:26   11:17:42 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 11:21:26   11:17:42 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 681ms (681ms including waiting). Image size: 194988415 bytes.
2026/09/07 11:21:26   11:17:42 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Started - Started container manager
2026/09/07 11:21:26   11:17:41 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 705ms (705ms including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Started - Started container manager
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Created - Created container: manager
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 456ms (456ms including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: AddedInterface - Add eth0 [10.132.0.85/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810"
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810" in 563ms (563ms including waiting). Image size: 173568461 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845" in 1.026s (1.026s including waiting). Image size: 168389130 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Started - Started container manager
2026/09/07 11:21:26   11:17:41 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" in 1.324s (1.324s including waiting). Image size: 158342259 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 625ms (625ms including waiting). Image size: 194988415 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:41 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Created - Created container: manager
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 611ms (611ms including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:40 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-78f79f4677-kqrgp to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75c8689979-mhl7v to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-module-controller-manager-59549b5854-sclh9 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-bvlbv to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-lnwt6 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: AddedInterface - Add eth0 [10.134.0.100/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Created - Created container: manager
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: AddedInterface - Add eth0 [10.132.0.84/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-bvlbv
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/spark-operator-module-controller-manager-59549b5854: SuccessfulCreate - Created pod: spark-operator-module-controller-manager-59549b5854-sclh9
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Started - Started container manager
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: AddedInterface - Add eth0 [10.134.0.98/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 301ms (301ms including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Created - Created container: manager
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-lnwt6
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Pulled - Container image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637" already present on machine
2026/09/07 11:21:26   11:17:40 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Pulling - Pulling image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35"
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/odh-notebook-controller-manager-78f79f4677: SuccessfulCreate - Created pod: odh-notebook-controller-manager-78f79f4677-kqrgp
2026/09/07 11:21:26   11:17:40 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: AddedInterface - Add eth0 [10.134.0.95/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:40 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845"
2026/09/07 11:21:26   11:17:40 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-78f79f4677 from 0 to 1
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/notebook-controller-deployment-75c8689979: SuccessfulCreate - Created pod: notebook-controller-deployment-75c8689979-mhl7v
2026/09/07 11:21:26   11:17:40 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75c8689979 from 0 to 1
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: AddedInterface - Add eth0 [10.134.0.96/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Started - Started container manager
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: AddedInterface - Add eth0 [10.134.0.99/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Deployment/spark-operator-module-controller-manager: ScalingReplicaSet - Scaled up replica set spark-operator-module-controller-manager-59549b5854 from 0 to 1
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: AddedInterface - Add eth0 [10.134.0.97/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:39 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-84glx to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:17:39 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-lqv8q to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:39 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-dqh6n to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:39 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-84glx
2026/09/07 11:21:26   11:17:39 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-lqv8q
2026/09/07 11:21:26   11:17:39 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-dqh6n
2026/09/07 11:21:26   11:17:39 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/07 11:21:26   11:17:38 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Started - Started container manager
2026/09/07 11:21:26   11:17:38 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9_d744adfe-3717-44d9-8ebd-4d4044f27ad6 became leader
2026/09/07 11:21:26   11:17:38 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Created - Created container: manager
2026/09/07 11:21:26   11:17:38 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4" in 2.552s (2.552s including waiting). Image size: 143076452 bytes.
2026/09/07 11:21:26   11:17:37 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:37 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:36 INFO Pod/workbenches-operator-c674d9c9-p42hk: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" already present on machine
2026/09/07 11:21:26   11:17:36 INFO Pod/workbenches-operator-c674d9c9-p42hk: Started - Started container manager
2026/09/07 11:21:26   11:17:36 INFO Pod/workbenches-operator-c674d9c9-p42hk: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:35 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 1.079s (1.079s including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 1.088s (1.088s including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4"
2026/09/07 11:21:26   11:17:35 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Started - Started container manager
2026/09/07 11:21:26   11:17:35 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: AddedInterface - Add eth0 [10.134.0.94/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:35 INFO Pod/workbenches-operator-c674d9c9-p42hk: Pulled - Successfully pulled image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" in 2.177s (2.177s including waiting). Image size: 185267738 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" in 2.244s (2.244s including waiting). Image size: 198393268 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Started - Started container manager
2026/09/07 11:21:26   11:17:35 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Started - Started container manager
2026/09/07 11:21:26   11:17:34 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:34 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-6b668bbbb: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9
2026/09/07 11:21:26   11:17:34 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:34 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-6b668bbbb from 0 to 1
2026/09/07 11:21:26   11:17:34 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:34 WARN Pod/rhods-dashboard-bfcc58b46-42vcs: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:17:34 WARN Pod/rhods-dashboard-bfcc58b46-42vcs: Unhealthy - Readiness probe failed: Get "https://10.133.0.52:8943/healthcheck": dial tcp 10.133.0.52:8943: connect: connection refused
2026/09/07 11:21:26   11:17:33 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Created - Created container: manager
2026/09/07 11:21:26   11:17:33 INFO Pod/workbenches-operator-c674d9c9-p42hk: Pulling - Pulling image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898"
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 598ms (598ms including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 517ms (517ms including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:33 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Started - Started container manager
2026/09/07 11:21:26   11:17:32 INFO Pod/workbenches-operator-c674d9c9-p42hk: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-c674d9c9-p42hk to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-nhrpr to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-vfz9x to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-jvq7v to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-2nlcj to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO Pod/workbenches-operator-c674d9c9-p42hk: AddedInterface - Add eth0 [10.134.0.93/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-c674d9c9 from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/workbenches-operator-c674d9c9: SuccessfulCreate - Created pod: workbenches-operator-c674d9c9-p42hk
2026/09/07 11:21:26   11:17:32 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-nhrpr
2026/09/07 11:21:26   11:17:32 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: AddedInterface - Add eth0 [10.134.0.92/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:32 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: AddedInterface - Add eth0 [10.134.0.90/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: AddedInterface - Add eth0 [10.134.0.91/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-vfz9x
2026/09/07 11:21:26   11:17:32 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Pulling - Pulling image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2"
2026/09/07 11:21:26   11:17:32 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-2nlcj
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-jvq7v
2026/09/07 11:21:26   11:17:32 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/07 11:21:26   11:17:32 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: AddedInterface - Add eth0 [10.132.0.83/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:30 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Created - Created container: manager
2026/09/07 11:21:26   11:17:30 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Pulled - Successfully pulled image "registry.k8s.io/pause:3.9" in 370ms (370ms including waiting). Image size: 750414 bytes.
2026/09/07 11:21:26   11:17:30 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Started - Started container manager
2026/09/07 11:21:26   11:17:29 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-6b8fb45c8b-hsr99 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:29 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-6b8fb45c8b from 0 to 1
2026/09/07 11:21:26   11:17:29 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: AddedInterface - Add eth0 [10.134.0.89/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:29 INFO ReplicaSet/feast-operator-controller-manager-6b8fb45c8b: SuccessfulCreate - Created pod: feast-operator-controller-manager-6b8fb45c8b-hsr99
2026/09/07 11:21:26   11:17:29 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Pulling - Pulling image "registry.k8s.io/pause:3.9"
2026/09/07 11:21:26   11:17:25 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:25 WARN Pod/rhods-dashboard-bfcc58b46-v2xm8: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:17:25 INFO Pod/dashboard-operator-669d6564bd-q49dz: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:25 INFO Pod/aihub-controller-manager-d877888b8-s4mh4: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:25 WARN Endpoints/aihub-controller-manager-metrics-service: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/aihub-controller-manager-metrics-service: Operation cannot be fulfilled on endpoints "aihub-controller-manager-metrics-service": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/aihub-controller-manager-metrics-service, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: a5481786-dca0-4fe8-b208-e35728ace0fb, UID in object meta: 
2026/09/07 11:21:26   11:17:24 INFO Pod/odh-observability-59db79cb46-jxh7c: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 INFO Pod/ai-gateway-operator-89f69845b-9gxl8: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 WARN Pod/kubeflow-trainer-controller-manager-5449979c85-h4tsl: Unhealthy - Readiness probe failed: Get "http://10.133.0.45:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/07 11:21:26   11:17:24 INFO Pod/kserve-module-controller-manager-66ff7c977-8cc4s: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 WARN Pod/rhods-dashboard-bfcc58b46-v2xm8: Unhealthy - Readiness probe failed: Get "https://10.132.0.55:8943/healthcheck": dial tcp 10.132.0.55:8943: connect: connection refused
2026/09/07 11:21:26   11:17:24 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-2chm7: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 INFO Pod/workbenches-operator-c674d9c9-8qhfq: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:23 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:23 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:23 INFO Pod/opendatahub-feast-operator-84ff599545-w98d9: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:23 INFO Pod/mlflow-operator-controller-manager-5bbb468f5f-4cnxp: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/notebook-controller-deployment-75c8689979-4mdp7: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/odh-notebook-controller-manager-5bcb7b86db-24424: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/spark-operator-module-controller-manager-59549b5854-v7bd8: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/opendatahub-ogx-operator-644d45d5b-cdxvd: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/odh-model-controller-6b9dd8b59d-gzhkt: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/spark-operator-controller-c594b967f-kgcc6: Killing - Stopping container controller
2026/09/07 11:21:26   11:17:21 INFO Pod/mlflow-ui-7bb6d46f5f-nkqj6: Killing - Stopping container mlflow-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-v2xm8: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-v2xm8: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-v2xm8: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:17:21 INFO Pod/model-registry-operator-controller-manager-6bcdfb5b87-ws2bb: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-42vcs: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-42vcs: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-42vcs: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:17:21 INFO Pod/notebooks-ui-6577cc467-vvp7j: Killing - Stopping container notebooks-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-kpjkz: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/dashboard-redirect-55f4444684-j8lnb: Killing - Stopping container nginx
2026/09/07 11:21:26   11:17:21 INFO Pod/model-registry-ui-67b8c946b9-s68np: Killing - Stopping container model-registry-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/kserve-controller-manager-5fbbd6dd88-zzwnk: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/gen-ai-ui-6459ffd979-sw7x5: Killing - Stopping container gen-ai-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/dashboard-redirect-55f4444684-htz29: Killing - Stopping container nginx
2026/09/07 11:21:26   11:17:21 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-f5p8j: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/automl-ui-74d9b97d49-x268c: Killing - Stopping container automl-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/autorag-ui-58f548cbd-jb9rf: Killing - Stopping container autorag-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/catalog-controller-manager-67d887945b-qlktz: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-pnjlp: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/spark-operator-webhook-5d466b7d66-9vxzx: Killing - Stopping container webhook
2026/09/07 11:21:26   11:17:21 INFO Pod/maas-ui-675f94cfd8-b6vdb: Killing - Stopping container maas-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/model-serving-api-7df8c9655f-hbdt2: Killing - Stopping container server
2026/09/07 11:21:26   11:17:21 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-h4tsl: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 WARN Pod/maas-controller-56c898799-t7zrv: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "maas-controller-webhook-cert" not found
2026/09/07 11:21:26   11:17:21 INFO Pod/maas-controller-56c898799-t7zrv: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 WARN Pod/maas-controller-56c898799-t7zrv: FailedMount - MountVolume.SetUp failed for volume "metrics-certs" : secret "maas-controller-metrics-tls" not found
2026/09/07 11:21:26   11:17:21 INFO Pod/agent-ops-ui-7df989bc55-tcsxh: Killing - Stopping container agent-ops-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/eval-hub-ui-869b89d9b4-zsgms: Killing - Stopping container eval-hub-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8976vqd: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:20 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:20 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:20 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:20 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-8djfr: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:20 INFO Pod/kuberay-operator-7d87f54798-z9q95: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:17:20 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-x9blj: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:20 WARN Pod/maas-controller-56c898799-t7zrv: FailedToRetrieveImagePullSecret - Unable to retrieve some image pull secrets (maas-controller-dockercfg-vp7c6); attempting to pull the image may not succeed.
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 INFO Pod/kserve-localmodelnode-agent-4z77m: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:18 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-4z77m
2026/09/07 11:21:26   11:17:18 INFO Pod/kserve-localmodelnode-agent-c5dnt: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:18 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-c5dnt
2026/09/07 11:21:26 === RESOURCE QUOTAS ===
2026/09/07 11:21:26 No resource quotas found in monitored namespaces
2026/09/07 11:21:26 Classification for ConfigMap deletion recovery: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/07 11:21:26   Evidence: event Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get \"https://10.132.0.101:8444/healthz\": dial tcp 10.132.0.101:8444: connect: connection refused"],"confidence":"medium"}
2026/09/07 11:21:26 Diagnostics complete for failed test: ConfigMap deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_localmodel-webhook-server-service" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_model-serving-api" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-metrics-service" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-webhook-service" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="155.660"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-controller-manager" classname="" time="35.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-localmodel-controller-manager" classname="" time="45.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_llmisvc-controller-manager" classname="" time="35.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_model-serving-api" classname="" time="15.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_odh-model-controller" classname="" time="25.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_disabled" classname="" time="10.350"></testcase>
		<testcase name="TestOdhOperator/components/group_2" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue" classname="" time="92.750"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry" classname="" time="90.640">
			<system-out><![CDATA[2026-09-07T11:24:43Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_enabled" classname="" time="50.250">
			<system-out><![CDATA[2026-09-07T11:24:43Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_error_with_ocp_kueue-operator_not_installed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_removed_to_unmanaged_transition" classname="" time="10.680">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Creating test namespace (test-kueue-managed-daf9tupve3cs7mgi5s70) with Kueue management annotation.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:156: Updating the management state of the component in the DataScienceCluster to Unmanaged with autoCreateQueues=true (expecting NotReady initially due to missing operator).
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Verifying the component in the DataScienceCluster transitions to Ready state.
    helper_test.go:156: Verifying Kueue configuration (cluster) is created with all expected frameworks.
    helper_test.go:156: Verifying default ClusterQueue and LocalQueue resources are created in namespace test-kueue-managed-daf9tupve3cs7mgi5s70.
    helper_test.go:156: Verifying default resource flavor (default-flavor) is created.
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind']]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_to_removed_transition" classname="" time="50.980">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Defining expected conditions for Unmanaged state (Ready=True).
    helper_test.go:156: Updating DSC default-dsc to set Kueue managementState to Unmanaged with autoCreateQueues=true.
    helper_test.go:156: Creating test namespace (test-kueue-managed-daf9u19ve3cs7mgi5s7g) with Kueue management annotation.
    helper_test.go:156: Verifying default Kueue resources exist in namespace test-kueue-managed-daf9u19ve3cs7mgi5s7g.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_module_CR_spec" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_registries_namespace_created" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_disabled" classname="" time="40.260">
			<system-out><![CDATA[    helper_test.go:156: Verifying default resource flavor (default-flavor) is created.
    helper_test.go:156: Updating DSC default-dsc to set Kueue managementState to Removed.
    helper_test.go:156: Verifying default resources are still present in namespace test-kueue-managed-daf9u19ve3cs7mgi5s7g.
    helper_test.go:156: Cleaning up Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_autoCreateQueues_disabled_skips_queue_creation" classname="" time="20.680">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Creating test namespace (test-kueue-no-queues-daf9ue1ve3cs7mgi5s80) with Kueue management annotation.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:156: Updating DSC to set Kueue to Unmanaged with autoCreateQueues=false (default).
    helper_test.go:156: Verifying Kueue configuration (cluster) is still created.
    helper_test.go:156: Verifying no ClusterQueue, ResourceFlavor or LocalQueue is created (autoCreateQueues=false).
    helper_test.go:156: Cleaning up Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind']]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No ConfigMap resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Service resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Deployment resources found for component Kueue, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_disabled" classname="" time="10.230">
			<system-out><![CDATA[    helper_test.go:156: Setting DSC component default-kueue to Removed state.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator" classname="" time="91.230"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai" classname="" time="294.100">
			<system-out><![CDATA[2026-09-07T11:26:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_enabled" classname="" time="80.900">
			<system-out><![CDATA[2026-09-07T11:26:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_enabled" classname="" time="111.380"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_disabled" classname="" time="10.190"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_update_operand_resources" classname="" time="20.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_update_operand_resources/deployment_trustyai-service-operator-controller-manager" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_MCP_guardrails_mode" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_pre_check" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery" classname="" time="141.910"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="111.560"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-dsc-config" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-config" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-coding-v1" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-instruction-following-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-leaderboard-v2" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-long-context-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-model-validation" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-open-telco-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-reasoning-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-safety-and-fairness-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-standard-llm-evals-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-toxicity-and-ethical-principles" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-deepeval" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-garak" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-garak-kfp" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-guidellm" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-ibm-clear" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-inspect" classname="" time="5.110"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-lighteval" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-lm-evaluation-harness" classname="" time="5.130"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-ragas" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-lmeval-tasks" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="15.220"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-controller-manager-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-webhook-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.120"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_trustyai-service-operator-controller-manager" classname="" time="15.100"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_disabled" classname="" time="20.540"></testcase>
		<testcase name="TestOdhOperator/components/group_4" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve" classname="" time="91.210"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kueue" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:762: Skipping: flaky due to namespace stuck in Terminating state. See RHOAIENG-46773 and RHOAIENG-46774.]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer" classname="" time="30.380"></testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer/Validate_component_enabled" classname="" time="30.210">
			<system-out><![CDATA[2026-09-07T11:31:10Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve/Validate_component_enabled" classname="" time="91.040"></testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer/Validate_external_operator_degraded_condition_monitoring" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve/Validate_external_operator_degraded_condition_monitoring" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache" classname="" time="101.240">
			<system-out><![CDATA[2026-09-07T11:32:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_component_enabled" classname="" time="101.100"></testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_ConfigMap" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services" classname="" time="0.220"></testcase>
		<testcase name="TestOdhOperator/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/services/group_1/auth" classname="" time="0.220"></testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway" classname="" time="0.160">
			<system-out><![CDATA[2026-09-07T11:34:22Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_Auth_system_initialization" classname="" time="0.090">
			<system-out><![CDATA[2026-09-07T11:34:22Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_GatewayConfig_creation" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: Validating GatewayConfig resource
    helper_test.go:156: GatewayConfig validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_infrastructure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_client_and_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_proxy_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_token_forwarding_to_dashboard" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_kube-auth-proxy_TLS_args_match_cluster_APIServer" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_HorizontalPodAutoscaler_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_NetworkPolicy_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_callback_HTTPRoute" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EnvoyFilter_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EDS_endpoint_discovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_ready_status" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: Validating Gateway ready status
    helper_test.go:156: Gateway ready status validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_dashboard_redirect_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_addition_of_RoleBinding_when_group_is_added" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_removal_of_bindings_when_a_group_is_removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_blocks_Auth_update_with_invalid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_allows_Auth_with_valid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests" classname="" time="0.130">
			<system-out><![CDATA[2026-09-07T11:34:23Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_operator_deployment_health" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_leader_election_behavior" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_success" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_failure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_missing_CRD_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_RBAC_restriction_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/V2_to_V3_upgrade_E2E_Tests" classname="" time="0.120">
			<skipped message="Skipped"><![CDATA[2026-09-07T11:34:23Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests" classname="" time="0.140">
			<system-out><![CDATA[2026-09-07T11:34:23Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_creation_of_configmap_with_deletion_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_that_owned_namespaces_are_not_deleted" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/upgrade_DSC_and_DSCI_v1_API" classname="" time="0.130">
			<skipped message="Skipped"><![CDATA[2026-09-07T11:34:23Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestLogPodsSectionContinuesWithPartialData" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesComponentsMatchRealTestGroup" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesServicesMatchRealTestGroup" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/space-separated_matches_the_real_contract" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/comma-separated_collapses_into_one_malformed_name_--_this_is_the_bug,_not_the_fix" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/single_name_has_no_separator_to_get_wrong" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/all_enabled_when_no_flags" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/explicit_inclusion_limits_to_named_tests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/exclusion_removes_from_full_set" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/inclusion_and_exclusion_combined" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test/monitoring" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:78: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:85: Test nonexistent is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:94: Test monitoring is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/auth" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/gateway" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:128: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:142: No tests remaining in group single after excluding only]]></skipped>
		</testcase>
		<testcase name="TestIsTestEnabled" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/enabled_by_default" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/disabled_by_exclusion_flag" classname="" time="0.000"></testcase>
		<system-out><![CDATA[    controller_test.go:762: Restoring operator to original version for Phase 3
    controller_test.go:762: Patched Subscription to remove env vars: [RHAI_VERSION CI]
    controller_test.go:762: Waiting for DSC to fully reconcile at original version
    dag_ordering_test.go:193: Cleanup: setting all components to Removed, deleting quota, and restoring operator env vars
    dag_ordering_test.go:196: Patched Subscription to remove env vars: [RHAI_VERSION CI]
2026/09/07 11:21:26 TEST FAILURE DETECTED: ConfigMap deletion recovery
2026/09/07 11:21:26 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:ConfigMap deletion recovery) ===
2026/09/07 11:21:26 TEST FAILURE DETECTED: Validate resource deletion recovery
2026/09/07 11:21:26 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:Validate resource deletion recovery) ===
2026/09/07 11:21:26 === CLUSTER STATE ===
2026/09/07 11:21:26 Node: ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:21:26   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:21:26   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:21:26   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:21:26   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 11:21:26 Node: ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:21:26   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:21:26   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:21:26   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:21:26   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:21:26 Node: ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:21:26   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:21:26   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:21:26   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:21:26   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:21:26 === DEPLOYMENTS ===
2026/09/07 11:21:26 Namespace: redhat-ods-applications
2026/09/07 11:21:26   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 11:21:26   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 11:21:26   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment kserve-controller-manager: 2/2 (READY)
2026/09/07 11:21:26   Deployment kserve-localmodel-controller-manager: 2/2 (READY)
2026/09/07 11:21:26   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/07 11:21:26   Deployment llmisvc-controller-manager: 2/2 (READY)
2026/09/07 11:21:26   Deployment maas-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment model-serving-api: 2/2 (READY)
2026/09/07 11:21:26   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 11:21:26   Deployment odh-model-controller: 2/2 (READY)
2026/09/07 11:21:26   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 11:21:26 Namespace: kube-system
2026/09/07 11:21:26 === PODS ===
2026/09/07 11:21:26 Namespace redhat-ods-applications: no problematic pods found
2026/09/07 11:21:26 Namespace kube-system: no problematic pods found
2026/09/07 11:21:26 === OPERATOR STATUS ===
2026/09/07 11:21:26 Operator deployment: 3/3 ready
2026/09/07 11:21:26 === DSCI STATUS ===
2026/09/07 11:21:26 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 11:21:26 === DSC STATUS ===
2026/09/07 11:21:26 DSC default-dsc:
2026/09/07 11:21:26   Ready: True - 
2026/09/07 11:21:26   ProvisioningSucceeded: True - 
2026/09/07 11:21:26   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 11:21:26   ModulesReady: True - 
2026/09/07 11:21:26   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:21:26   DashboardReady: True - 
2026/09/07 11:21:26   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   KserveLLMInferenceServiceDependencies: True - 
2026/09/07 11:21:26   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/07 11:21:26   KserveReady: True - 
2026/09/07 11:21:26   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:21:26   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   RayReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   TrainerReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 11:21:26   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 11:21:26   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 11:21:26 === RECENT EVENTS ===
2026/09/07 11:21:26   11:21:23 WARN Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
2026/09/07 11:21:26   11:21:16 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:16 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulDelete - Deleted pod: rhods-dashboard-7f94fcf99d-6cjmj
2026/09/07 11:21:26   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:21:16 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7f94fcf99d from 1 to 0
2026/09/07 11:21:26   11:21:11 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:11 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:21:05 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe failed: Get "https://10.133.0.111:8444/healthz": dial tcp 10.133.0.111:8444: connect: connection refused
2026/09/07 11:21:26   11:20:55 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:20:55 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe failed: Get "https://10.133.0.111:8943/healthcheck": dial tcp 10.133.0.111:8943: connect: connection refused
2026/09/07 11:21:26   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 388ms (388ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: core-bff
2026/09/07 11:21:26   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container core-bff
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 594ms (594ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-775867476c-f7trw: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-f7trw to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:20:36 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-775867476c from 1 to 2
2026/09/07 11:21:26   11:20:36 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulDelete - Deleted pod: rhods-dashboard-5c9d44b7b4-dgpl5
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:20:36 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-5c9d44b7b4 from 1 to 0
2026/09/07 11:21:26   11:20:36 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-f7trw
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 2.22s (2.22s including waiting). Image size: 239132625 bytes.
2026/09/07 11:21:26   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Started - Started container manager
2026/09/07 11:21:26   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Created - Created container: manager
2026/09/07 11:21:26   11:20:31 INFO Pod/odh-model-controller-57457c4c89-bgwgw: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/07 11:21:26   11:20:31 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:21:26   11:20:30 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-bgwgw to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:20:30 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/07 11:21:26   11:20:30 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-bgwgw
2026/09/07 11:21:26   11:20:27 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe failed: Get "https://10.133.0.114:8943/healthcheck": dial tcp 10.133.0.114:8943: connect: connection refused
2026/09/07 11:21:26   11:20:27 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:20:23 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe failed: Get "https://10.133.0.114:8444/healthz": dial tcp 10.133.0.114:8444: connect: connection refused
2026/09/07 11:21:26   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Created - Created container: server
2026/09/07 11:21:26   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.284s (1.284s including waiting). Image size: 172459788 bytes.
2026/09/07 11:21:26   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Started - Started container server
2026/09/07 11:21:26   11:20:11 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 11:21:26   11:20:11 INFO Pod/model-serving-api-7df8c9655f-dz7bh: AddedInterface - Add eth0 [10.134.0.112/23] from ovn-kubernetes
2026/09/07 11:21:26   11:20:10 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-dz7bh to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:20:10 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-dz7bh
2026/09/07 11:21:26   11:20:10 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/07 11:21:26   11:20:01 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe failed: Get "https://10.134.0.106:8444/healthz": dial tcp 10.134.0.106:8444: connect: connection refused
2026/09/07 11:21:26   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container core-bff
2026/09/07 11:21:26   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: core-bff
2026/09/07 11:21:26   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 373ms (373ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: AddedInterface - Add eth0 [10.134.0.111/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 429ms (429ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-775867476c-bpspj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-bpspj to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:57 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulDelete - Deleted pod: rhods-dashboard-7f94fcf99d-rwzbj
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:19:57 INFO Pod/autorag-ui-58f548cbd-8pn4g: Killing - Stopping container autorag-ui
2026/09/07 11:21:26   11:19:57 WARN Endpoints/odh-dashboard-automl-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-automl-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-automl-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-automl-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 9931d5c4-97af-4cd3-9b95-ee5288a56884, UID in object meta: 
2026/09/07 11:21:26   11:19:57 WARN Endpoints/odh-dashboard-autorag-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-autorag-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-autorag-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-autorag-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 79bb98aa-3dac-4c9d-ae1e-46c4f3573e85, UID in object meta: 
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:19:57 INFO Pod/automl-ui-74d9b97d49-lzkxj: Killing - Stopping container automl-ui
2026/09/07 11:21:26   11:19:57 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-775867476c from 0 to 1
2026/09/07 11:21:26   11:19:57 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7f94fcf99d from 2 to 1
2026/09/07 11:21:26   11:19:57 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-bpspj
2026/09/07 11:21:26   11:19:56 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe failed: Get "https://10.134.0.106:8943/healthcheck": dial tcp 10.134.0.106:8943: connect: connection refused
2026/09/07 11:21:26   11:19:56 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Created - Created container: manager
2026/09/07 11:21:26   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 1.941s (1.941s including waiting). Image size: 235279660 bytes.
2026/09/07 11:21:26   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Started - Started container manager
2026/09/07 11:21:26   11:19:51 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:21:26   11:19:51 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: AddedInterface - Add eth0 [10.132.0.104/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:50 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-hvhs7 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:19:50 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-hvhs7
2026/09/07 11:21:26   11:19:50 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/07 11:21:26   11:19:46 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:19:46 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:19:46 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:41 WARN Pod/spark-operator-webhook-5d466b7d66-g8r7p: Unhealthy - Readiness probe failed: Get "http://10.133.0.109:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/07 11:21:26   11:19:41 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:40 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Killing - Stopping container controller
2026/09/07 11:21:26   11:19:40 WARN Pod/spark-operator-controller-c594b967f-wpdtn: Unhealthy - Readiness probe failed: Get "http://10.133.0.108:8081/readyz": dial tcp 10.133.0.108:8081: connect: connection refused
2026/09/07 11:21:26   11:19:40 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Killing - Stopping container webhook
2026/09/07 11:21:26   11:19:36 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Created - Created container: core-bff
2026/09/07 11:21:26   11:19:36 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Started - Started container core-bff
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 391ms (391ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 579ms (579ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:19:35 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7f94fcf99d-rwzbj to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:19:34 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulDelete - Deleted pod: rhods-dashboard-5c9d44b7b4-zhmtt
2026/09/07 11:21:26   11:19:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-5c9d44b7b4 from 2 to 1
2026/09/07 11:21:26   11:19:34 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7f94fcf99d from 1 to 2
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: AddedInterface - Add eth0 [10.133.0.114/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:19:34 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulCreate - Created pod: rhods-dashboard-7f94fcf99d-rwzbj
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:19:34 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:19:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Created - Created container: manager
2026/09/07 11:21:26   11:19:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Started - Started container manager
2026/09/07 11:21:26   11:19:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Started - Started container manager
2026/09/07 11:21:26   11:19:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 2.448s (2.448s including waiting). Image size: 244249302 bytes.
2026/09/07 11:21:26   11:19:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.358s (1.358s including waiting). Image size: 178395693 bytes.
2026/09/07 11:21:26   11:19:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Created - Created container: manager
2026/09/07 11:21:26   11:19:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f897kgkm to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:31 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: AddedInterface - Add eth0 [10.132.0.102/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:31 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:19:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 11:21:26   11:19:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: AddedInterface - Add eth0 [10.134.0.110/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:31 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f897kgkm
2026/09/07 11:21:26   11:19:31 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:21:26   11:19:30 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-4lrwg to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:19:30 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:30 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 11:21:26   11:19:30 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-4lrwg
2026/09/07 11:21:26   11:19:30 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 1 to 2
2026/09/07 11:21:26   11:19:30 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Killing - Stopping container manager
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-gzzrx to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: AddedInterface - Add eth0 [10.134.0.109/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Created - Created container: kuberay-operator
2026/09/07 11:21:26   11:19:16 INFO Pod/kuberay-operator-7d87f54798-gzzrx: Started - Started container kuberay-operator
2026/09/07 11:21:26   11:19:16 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-gzzrx
2026/09/07 11:21:26   11:19:16 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 11:21:26   11:19:15 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:19:15 INFO Pod/kuberay-operator-7d87f54798-z4s96: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:19:13 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Started - Started container manager
2026/09/07 11:21:26   11:19:13 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Created - Created container: manager
2026/09/07 11:21:26   11:19:13 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 2.189s (2.189s including waiting). Image size: 247667445 bytes.
2026/09/07 11:21:26   11:19:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:21:26   11:19:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: AddedInterface - Add eth0 [10.133.0.113/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:10 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-v76tn to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:19:10 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-v76tn
2026/09/07 11:21:26   11:19:10 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 1 to 2
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: Started - Started container manager
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" already present on machine
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: Created - Created container: manager
2026/09/07 11:21:26   11:19:01 INFO Pod/dashboard-operator-669d6564bd-gcllm: AddedInterface - Add eth0 [10.134.0.108/23] from ovn-kubernetes
2026/09/07 11:21:26   11:19:00 INFO Pod/dashboard-operator-669d6564bd-gcllm: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-gcllm to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:19:00 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-gcllm
2026/09/07 11:21:26   11:19:00 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 1 to 2
2026/09/07 11:21:26   11:18:56 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Created - Created container: core-bff
2026/09/07 11:21:26   11:18:56 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 364ms (364ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:18:56 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Started - Started container core-bff
2026/09/07 11:21:26   11:18:55 INFO Pod/automl-ui-74d9b97d49-lzkxj: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af" in 3.59s (3.59s including waiting). Image size: 222806700 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/automl-ui-74d9b97d49-lzkxj: Started - Started container automl-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:18:55 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3" in 3.473s (3.473s including waiting). Image size: 184646446 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 2.311s (2.311s including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:18:55 INFO Pod/notebooks-ui-6577cc467-6fxnb: Started - Started container notebooks-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Started - Started container agent-ops-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:18:55 INFO Pod/notebooks-ui-6577cc467-6fxnb: Created - Created container: notebooks-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/notebooks-ui-6577cc467-6fxnb: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003" in 3.13s (3.13s including waiting). Image size: 190593254 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/autorag-ui-58f548cbd-8pn4g: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2" in 3.257s (3.257s including waiting). Image size: 223947455 bytes.
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:18:55 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:18:55 INFO Pod/automl-ui-74d9b97d49-lzkxj: Created - Created container: automl-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/autorag-ui-58f548cbd-8pn4g: Created - Created container: autorag-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Created - Created container: agent-ops-ui
2026/09/07 11:21:26   11:18:55 INFO Pod/autorag-ui-58f548cbd-8pn4g: Started - Started container autorag-ui
2026/09/07 11:21:26   11:18:53 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:18:52 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7f94fcf99d-6cjmj to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:52 INFO Pod/notebooks-ui-6577cc467-6fxnb: AddedInterface - Add eth0 [10.132.0.100/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:52 INFO Pod/notebooks-ui-6577cc467-6fxnb: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003"
2026/09/07 11:21:26   11:18:52 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7f94fcf99d from 0 to 1
2026/09/07 11:21:26   11:18:52 INFO Pod/autorag-ui-58f548cbd-8pn4g: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2"
2026/09/07 11:21:26   11:18:52 INFO Pod/autorag-ui-58f548cbd-8pn4g: AddedInterface - Add eth0 [10.132.0.98/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:52 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulCreate - Created pod: rhods-dashboard-7f94fcf99d-6cjmj
2026/09/07 11:21:26   11:18:52 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: AddedInterface - Add eth0 [10.132.0.101/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/notebooks-ui-6577cc467-6fxnb: Scheduled - Successfully assigned redhat-ods-applications/notebooks-ui-6577cc467-6fxnb to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:51 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-7df989bc55-wh2xr to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:51 INFO Pod/maas-ui-675f94cfd8-57bsm: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe" in 442ms (442ms including waiting). Image size: 189581000 bytes.
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/07 11:21:26   11:18:51 WARN Pod/autorag-ui-58f548cbd-8pn4g: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "autorag-proxy-tls" not found
2026/09/07 11:21:26   11:18:51 INFO ReplicaSet/notebooks-ui-6577cc467: SuccessfulCreate - Created pod: notebooks-ui-6577cc467-6fxnb
2026/09/07 11:21:26   11:18:51 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-8j6h9_e619c468-c35f-48a9-8f85-59edaf9e7466 became leader
2026/09/07 11:21:26   11:18:51 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-cszzw_b4119a50-069e-4b0b-8e87-25a733e64a6a became leader
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: AddedInterface - Add eth0 [10.133.0.112/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/maas-ui-675f94cfd8-57bsm: Created - Created container: maas-ui
2026/09/07 11:21:26   11:18:51 INFO Deployment/notebooks-ui: ScalingReplicaSet - Scaled up replica set notebooks-ui-6577cc467 from 0 to 1
2026/09/07 11:21:26   11:18:51 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Started - Started container gen-ai-ui
2026/09/07 11:21:26   11:18:51 INFO Pod/automl-ui-74d9b97d49-lzkxj: AddedInterface - Add eth0 [10.132.0.97/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Created - Created container: manager
2026/09/07 11:21:26   11:18:51 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-7df989bc55 from 0 to 1
2026/09/07 11:21:26   11:18:51 INFO Pod/maas-ui-675f94cfd8-57bsm: Started - Started container maas-ui
2026/09/07 11:21:26   11:18:51 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: AddedInterface - Add eth0 [10.132.0.99/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Started - Started container manager
2026/09/07 11:21:26   11:18:51 INFO Pod/automl-ui-74d9b97d49-lzkxj: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af"
2026/09/07 11:21:26   11:18:51 INFO Pod/agent-ops-ui-7df989bc55-wh2xr: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3"
2026/09/07 11:21:26   11:18:51 INFO ReplicaSet/agent-ops-ui-7df989bc55: SuccessfulCreate - Created pod: agent-ops-ui-7df989bc55-wh2xr
2026/09/07 11:21:26   11:18:50 INFO Pod/autorag-ui-58f548cbd-8pn4g: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-58f548cbd-8pn4g to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:50 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89f5np4: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89f5np4 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:50 INFO Pod/automl-ui-74d9b97d49-lzkxj: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-74d9b97d49-lzkxj to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:50 INFO Pod/maas-ui-675f94cfd8-57bsm: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-675f94cfd8-57bsm to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:50 WARN Pod/automl-ui-74d9b97d49-lzkxj: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/07 11:21:26   11:18:50 INFO Pod/maas-ui-675f94cfd8-57bsm: AddedInterface - Add eth0 [10.132.0.96/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:50 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-wrr9b_fbc23ba2-8d7b-45c8-9965-936c02dedcd8 became leader
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89f5np4
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/autorag-ui-58f548cbd: SuccessfulCreate - Created pod: autorag-ui-58f548cbd-8pn4g
2026/09/07 11:21:26   11:18:50 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-58f548cbd from 0 to 1
2026/09/07 11:21:26   11:18:50 INFO Pod/maas-ui-675f94cfd8-57bsm: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe"
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/maas-ui-675f94cfd8: SuccessfulCreate - Created pod: maas-ui-675f94cfd8-57bsm
2026/09/07 11:21:26   11:18:50 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 1 to 2
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: AddedInterface - Add eth0 [10.134.0.107/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:50 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-74d9b97d49 from 0 to 1
2026/09/07 11:21:26   11:18:50 INFO ReplicaSet/automl-ui-74d9b97d49: SuccessfulCreate - Created pod: automl-ui-74d9b97d49-lzkxj
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987"
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987" in 477ms (477ms including waiting). Image size: 238860397 bytes.
2026/09/07 11:21:26   11:18:50 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Created - Created container: gen-ai-ui
2026/09/07 11:21:26   11:18:49 INFO Pod/gen-ai-ui-6459ffd979-lqcdt: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-6459ffd979-lqcdt to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:49 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-6459ffd979 from 0 to 1
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Started - Started container core-bff
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Started - Started container core-bff
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Created - Created container: core-bff
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 435ms (435ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:18:49 INFO ReplicaSet/gen-ai-ui-6459ffd979: SuccessfulCreate - Created pod: gen-ai-ui-6459ffd979-lqcdt
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 421ms (421ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:21:26   11:18:49 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-675f94cfd8 from 0 to 1
2026/09/07 11:21:26   11:18:49 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Created - Created container: core-bff
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: AddedInterface - Add eth0 [10.134.0.106/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Started - Started container rhods-dashboard
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 467ms (467ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 596ms (597ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Started - Started container kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Created - Created container: kube-rbac-proxy
2026/09/07 11:21:26   11:18:48 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Created - Created container: rhods-dashboard
2026/09/07 11:21:26   11:18:47 INFO Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-5c9d44b7b4-zhmtt to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:47 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-5c9d44b7b4-dgpl5 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:47 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulCreate - Created pod: rhods-dashboard-5c9d44b7b4-zhmtt
2026/09/07 11:21:26   11:18:47 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulCreate - Created pod: rhods-dashboard-5c9d44b7b4-dgpl5
2026/09/07 11:21:26   11:18:47 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/07 11:21:26   11:18:47 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: AddedInterface - Add eth0 [10.133.0.111/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:47 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-5c9d44b7b4 from 0 to 2
2026/09/07 11:21:26   11:18:43 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7d87f54798-z4s96_bbd1a190-4af4-4ea7-a6dc-91a18c29f0fd became leader
2026/09/07 11:21:26   11:18:43 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-669d6564bd-twg9s_17baf0a1-3f70-4589-8a23-c2e37a314d8a became leader
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: AddedInterface - Add eth0 [10.134.0.105/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Created - Created container: kuberay-operator
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Started - Started container kuberay-operator
2026/09/07 11:21:26   11:18:41 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 11:21:26   11:18:40 INFO Pod/kuberay-operator-7d87f54798-6pzmm: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-6pzmm to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:40 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 1 to 2
2026/09/07 11:21:26   11:18:40 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-6pzmm
2026/09/07 11:21:26   11:18:35 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Started - Started container server
2026/09/07 11:21:26   11:18:35 INFO Pod/kserve-localmodelnode-agent-4psfs: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 1.09s (1.09s including waiting). Image size: 244000959 bytes.
2026/09/07 11:21:26   11:18:35 INFO Pod/kserve-localmodelnode-agent-4psfs: Created - Created container: manager
2026/09/07 11:21:26   11:18:35 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Created - Created container: server
2026/09/07 11:21:26   11:18:35 INFO Pod/kserve-localmodelnode-agent-4psfs: Started - Started container manager
2026/09/07 11:21:26   11:18:35 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Started - Started container manager
2026/09/07 11:21:26   11:18:35 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Created - Created container: manager
2026/09/07 11:21:26   11:18:35 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.706s (1.706s including waiting). Image size: 178395693 bytes.
2026/09/07 11:21:26   11:18:35 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 2.128s (2.128s including waiting). Image size: 172459788 bytes.
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Created - Created container: manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-4psfs: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:21:26   11:18:34 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Started - Started container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-c65mt: Started - Started container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 505ms (505ms including waiting). Image size: 247667445 bytes.
2026/09/07 11:21:26   11:18:34 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Created - Created container: manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-4psfs: AddedInterface - Add eth0 [10.132.0.95/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-c65mt: Created - Created container: manager
2026/09/07 11:21:26   11:18:34 INFO Pod/kserve-localmodelnode-agent-c65mt: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 720ms (720ms including waiting). Image size: 244000959 bytes.
2026/09/07 11:21:26   11:18:34 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Started - Started container manager
2026/09/07 11:21:26   11:18:34 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 527ms (527ms including waiting). Image size: 239132625 bytes.
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-c65mt: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-c65mt to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-4psfs: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-4psfs to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Started - Started container manager
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 484ms (484ms including waiting). Image size: 244249302 bytes.
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-c65mt: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:21:26   11:18:33 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-4psfs
2026/09/07 11:21:26   11:18:33 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:21:26   11:18:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 593ms (593ms including waiting). Image size: 235279660 bytes.
2026/09/07 11:21:26   11:18:33 INFO Pod/odh-model-controller-57457c4c89-8j6h9: AddedInterface - Add eth0 [10.132.0.93/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Created - Created container: manager
2026/09/07 11:21:26   11:18:33 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:21:26   11:18:33 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: AddedInterface - Add eth0 [10.134.0.102/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:33 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-c65mt
2026/09/07 11:21:26   11:18:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: AddedInterface - Add eth0 [10.132.0.94/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:33 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 11:21:26   11:18:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Created - Created container: manager
2026/09/07 11:21:26   11:18:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Started - Started container manager
2026/09/07 11:21:26   11:18:33 INFO Pod/kserve-localmodelnode-agent-c65mt: AddedInterface - Add eth0 [10.134.0.104/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89fwzvr: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89fwzvr to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:32 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:32 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:32 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-8j6h9 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-v8wbk to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-cszzw to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-rgvs7 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-wrr9b to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: AddedInterface - Add eth0 [10.134.0.103/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-8j6h9
2026/09/07 11:21:26   11:18:32 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 11:21:26   11:18:32 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "run-as-ray-user": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": For
bidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-wrr9b
2026/09/07 11:21:26   11:18:32 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89fwzvr
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-rgvs7
2026/09/07 11:21:26   11:18:32 INFO Pod/model-serving-api-7df8c9655f-v8wbk: AddedInterface - Add eth0 [10.132.0.92/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: AddedInterface - Add eth0 [10.133.0.110/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:32 WARN Pod/odh-model-controller-57457c4c89-8j6h9: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found
2026/09/07 11:21:26   11:18:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:21:26   11:18:32 WARN Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kserve-webhook-server-cert" not found
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-v8wbk
2026/09/07 11:21:26   11:18:32 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 11:21:26   11:18:32 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-cszzw
2026/09/07 11:21:26   11:18:32 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "run-as-ray-user": Forbidden: not usable by user or serviceaccount, provider "openshift-ai-inferenceservice-image-volume-scc": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceac
count, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:21:26   11:18:30 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:29 INFO Pod/maas-controller-56c898799-lnmx8: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:26 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:26 INFO Lease/spark-operator-controller-lock: LeaderElection - spark-operator-controller-c594b967f-wpdtn_4159f6ac-8b27-4ac6-b4df-23a5801dff94 became leader
2026/09/07 11:21:26   11:18:25 INFO Pod/dashboard-operator-669d6564bd-twg9s: Started - Started container manager
2026/09/07 11:21:26   11:18:25 INFO Pod/dashboard-operator-669d6564bd-twg9s: Created - Created container: manager
2026/09/07 11:21:26   11:18:25 INFO Pod/dashboard-operator-669d6564bd-twg9s: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" in 1.431s (1.431s including waiting). Image size: 170934264 bytes.
2026/09/07 11:21:26   11:18:24 INFO Pod/dashboard-operator-669d6564bd-twg9s: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520"
2026/09/07 11:21:26   11:18:24 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-5d466b7d66-g8r7p_cc4b67b7-4769-455e-8ae2-4fa8a45217e9 became leader
2026/09/07 11:21:26   11:18:24 INFO Pod/dashboard-operator-669d6564bd-twg9s: AddedInterface - Add eth0 [10.132.0.91/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:24 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:24 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-59777cdbbb-stntf_4d39fb2b-23e8-4a3b-8da3-465cf3cd2dd3 became leader
2026/09/07 11:21:26   11:18:23 INFO Pod/dashboard-operator-669d6564bd-twg9s: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-twg9s to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:23 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-twg9s
2026/09/07 11:21:26   11:18:23 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 0 to 1
2026/09/07 11:21:26   11:18:22 INFO Pod/kuberay-operator-7d87f54798-z4s96: Started - Started container kuberay-operator
2026/09/07 11:21:26   11:18:22 INFO Pod/kuberay-operator-7d87f54798-z4s96: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.937s (1.937s including waiting). Image size: 189730715 bytes.
2026/09/07 11:21:26   11:18:22 INFO Pod/kuberay-operator-7d87f54798-z4s96: Created - Created container: kuberay-operator
2026/09/07 11:21:26   11:18:22 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: AddedInterface - Add eth0 [10.134.0.101/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Started - Started container nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Created - Created container: nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Created - Created container: nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Started - Started container nginx
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-bs9jf: AddedInterface - Add eth0 [10.132.0.90/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:21 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 11:21:26   11:18:20 INFO Pod/dashboard-redirect-55f4444684-zpwrc: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-55f4444684-zpwrc to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:18:20 INFO Pod/dashboard-redirect-55f4444684-bs9jf: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-55f4444684-bs9jf to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:20 INFO Pod/kuberay-operator-7d87f54798-z4s96: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/07 11:21:26   11:18:20 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-55f4444684 from 0 to 2
2026/09/07 11:21:26   11:18:20 INFO Pod/kuberay-operator-7d87f54798-z4s96: AddedInterface - Add eth0 [10.132.0.89/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:20 INFO ReplicaSet/dashboard-redirect-55f4444684: SuccessfulCreate - Created pod: dashboard-redirect-55f4444684-bs9jf
2026/09/07 11:21:26   11:18:20 INFO ReplicaSet/dashboard-redirect-55f4444684: SuccessfulCreate - Created pod: dashboard-redirect-55f4444684-zpwrc
2026/09/07 11:21:26   11:18:19 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:19 INFO Pod/kuberay-operator-7d87f54798-z4s96: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-z4s96 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:19 WARN Pod/kuberay-operator-7d87f54798-z4s96: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/07 11:21:26   11:18:19 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:19 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-z4s96
2026/09/07 11:21:26   11:18:19 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 11:21:26   11:18:17 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:17 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:16 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:16 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:15 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:11 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Started - Started container manager
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-5fc6bb8d67-c4sfm to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Created - Created container: manager
2026/09/07 11:21:26   11:18:10 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-5fc6bb8d67 from 0 to 1
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: AddedInterface - Add eth0 [10.132.0.88/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:10 INFO ReplicaSet/feast-operator-controller-manager-5fc6bb8d67: SuccessfulCreate - Created pod: feast-operator-controller-manager-5fc6bb8d67-c4sfm
2026/09/07 11:21:26   11:18:10 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:10 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-c4sfm: Pulled - Container image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7" already present on machine
2026/09/07 11:21:26   11:18:10 INFO Pod/maas-controller-56c898799-lnmx8: Started - Started container manager
2026/09/07 11:21:26   11:18:10 INFO Pod/maas-controller-56c898799-lnmx8: Created - Created container: manager
2026/09/07 11:21:26   11:18:10 INFO Pod/maas-controller-56c898799-lnmx8: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd" in 2.095s (2.095s including waiting). Image size: 197531122 bytes.
2026/09/07 11:21:26   11:18:09 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Created - Created container: manager
2026/09/07 11:21:26   11:18:09 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2" in 1.707s (1.707s including waiting). Image size: 193888070 bytes.
2026/09/07 11:21:26   11:18:09 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Started - Started container manager
2026/09/07 11:21:26   11:18:08 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:08 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-webhook-5d466b7d66-g8r7p to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-controller-c594b967f-wpdtn to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Created - Created container: webhook
2026/09/07 11:21:26   11:18:08 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Started - Started container webhook
2026/09/07 11:21:26   11:18:08 INFO ReplicaSet/spark-operator-controller-c594b967f: SuccessfulCreate - Created pod: spark-operator-controller-c594b967f-wpdtn
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Started - Started container controller
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 11:21:26   11:18:08 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:08 INFO ReplicaSet/spark-operator-webhook-5d466b7d66: SuccessfulCreate - Created pod: spark-operator-webhook-5d466b7d66-g8r7p
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: AddedInterface - Add eth0 [10.133.0.108/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-webhook-5d466b7d66-g8r7p: AddedInterface - Add eth0 [10.133.0.109/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:08 INFO Pod/maas-controller-56c898799-lnmx8: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd"
2026/09/07 11:21:26   11:18:08 INFO Pod/maas-controller-56c898799-lnmx8: AddedInterface - Add eth0 [10.132.0.87/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Created - Created container: controller
2026/09/07 11:21:26   11:18:08 INFO Pod/spark-operator-controller-c594b967f-wpdtn: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 11:21:26   11:18:07 INFO Pod/maas-controller-56c898799-lnmx8: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-56c898799-lnmx8 to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:07 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-59777cdbbb-stntf to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:18:07 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-59777cdbbb from 0 to 1
2026/09/07 11:21:26   11:18:07 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: AddedInterface - Add eth0 [10.132.0.86/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:07 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/07 11:21:26   11:18:07 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-56c898799 from 0 to 1
2026/09/07 11:21:26   11:18:07 INFO ReplicaSet/ogx-k8s-operator-controller-manager-59777cdbbb: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-59777cdbbb-stntf
2026/09/07 11:21:26   11:18:07 INFO ReplicaSet/maas-controller-56c898799: SuccessfulCreate - Created pod: maas-controller-56c898799-lnmx8
2026/09/07 11:21:26   11:18:07 INFO Deployment/spark-operator-controller: ScalingReplicaSet - Scaled up replica set spark-operator-controller-c594b967f from 0 to 1
2026/09/07 11:21:26   11:18:07 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-stntf: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2"
2026/09/07 11:21:26   11:18:05 INFO Pod/workbenches-operator-c674d9c9-p42hk: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:04 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Created - Created container: manager
2026/09/07 11:21:26   11:18:04 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5_757d5306-0f25-47ed-8a65-f3953293ce91 became leader
2026/09/07 11:21:26   11:18:04 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Started - Started container manager
2026/09/07 11:21:26   11:18:04 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728" in 2.346s (2.346s including waiting). Image size: 186574763 bytes.
2026/09/07 11:21:26   11:18:02 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-84glx_683221f7-bd93-4938-8e21-01ee7cd15381 became leader
2026/09/07 11:21:26   11:18:01 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:01 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:18:01 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5 to ip-10-0-142-230.ec2.internal
2026/09/07 11:21:26   11:18:01 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-5f798b84bf from 0 to 1
2026/09/07 11:21:26   11:18:01 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728"
2026/09/07 11:21:26   11:18:01 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:01 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Killing - Stopping container manager
2026/09/07 11:21:26   11:18:01 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-5f798b84bf: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5
2026/09/07 11:21:26   11:18:01 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-b5mh5: AddedInterface - Add eth0 [10.133.0.107/23] from ovn-kubernetes
2026/09/07 11:21:26   11:18:00 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-644d45d5b-bvlbv_807ffc24-612b-4923-8d3d-45464ff69e83 became leader
2026/09/07 11:21:26   11:17:59 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-dqh6n_b9bab43a-f849-4207-81cf-484344a0a195 became leader
2026/09/07 11:21:26   11:17:59 INFO Lease/opendatahub-feast-operator-lock: LeaderElection - opendatahub-feast-operator-84ff599545-lnwt6_77e2eb9d-3227-4929-b86a-065587d1ed38 became leader
2026/09/07 11:21:26   11:17:59 INFO Lease/spark-operator-module-controller-manager: LeaderElection - spark-operator-module-controller-manager-59549b5854-sclh9_81261d89-5699-4c77-8e2c-5fcb011b0d28 became leader
2026/09/07 11:21:26   11:17:58 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-89f69845b-lqv8q_2720f88b-2549-477e-a17f-7cd0e599056f became leader
2026/09/07 11:21:26   11:17:49 WARN Pod/rhods-dashboard-bfcc58b46-v2xm8: Unhealthy - Readiness probe failed: Get "https://10.132.0.55:8444/healthz": dial tcp 10.132.0.55:8444: connect: connection refused
2026/09/07 11:21:26   11:17:49 WARN Pod/rhods-dashboard-bfcc58b46-42vcs: Unhealthy - Readiness probe failed: Get "https://10.133.0.52:8444/healthz": dial tcp 10.133.0.52:8444: connect: connection refused
2026/09/07 11:21:26   11:17:43 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Started - Started container manager
2026/09/07 11:21:26   11:17:43 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Started - Started container manager
2026/09/07 11:21:26   11:17:42 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Started - Started container manager
2026/09/07 11:21:26   11:17:42 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Created - Created container: manager
2026/09/07 11:21:26   11:17:42 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 11:21:26   11:17:42 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 681ms (681ms including waiting). Image size: 194988415 bytes.
2026/09/07 11:21:26   11:17:42 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Started - Started container manager
2026/09/07 11:21:26   11:17:41 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 705ms (705ms including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Started - Started container manager
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Created - Created container: manager
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 456ms (456ms including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: AddedInterface - Add eth0 [10.132.0.85/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810"
2026/09/07 11:21:26   11:17:41 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810" in 563ms (563ms including waiting). Image size: 173568461 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845" in 1.026s (1.026s including waiting). Image size: 168389130 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Started - Started container manager
2026/09/07 11:21:26   11:17:41 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" in 1.324s (1.324s including waiting). Image size: 158342259 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 625ms (625ms including waiting). Image size: 194988415 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:41 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Created - Created container: manager
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 611ms (611ms including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:41 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:40 INFO Pod/odh-notebook-controller-manager-78f79f4677-kqrgp: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-78f79f4677-kqrgp to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75c8689979-mhl7v to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-module-controller-manager-59549b5854-sclh9 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-bvlbv to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-lnwt6 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:40 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: AddedInterface - Add eth0 [10.134.0.100/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Created - Created container: manager
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: AddedInterface - Add eth0 [10.132.0.84/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-bvlbv
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/spark-operator-module-controller-manager-59549b5854: SuccessfulCreate - Created pod: spark-operator-module-controller-manager-59549b5854-sclh9
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Started - Started container manager
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-ogx-operator-644d45d5b-bvlbv: AddedInterface - Add eth0 [10.134.0.98/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 301ms (301ms including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Created - Created container: manager
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-lnwt6
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: Pulled - Container image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637" already present on machine
2026/09/07 11:21:26   11:17:40 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Pulling - Pulling image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35"
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/odh-notebook-controller-manager-78f79f4677: SuccessfulCreate - Created pod: odh-notebook-controller-manager-78f79f4677-kqrgp
2026/09/07 11:21:26   11:17:40 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: AddedInterface - Add eth0 [10.134.0.95/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:40 INFO Pod/notebook-controller-deployment-75c8689979-mhl7v: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845"
2026/09/07 11:21:26   11:17:40 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-78f79f4677 from 0 to 1
2026/09/07 11:21:26   11:17:40 INFO ReplicaSet/notebook-controller-deployment-75c8689979: SuccessfulCreate - Created pod: notebook-controller-deployment-75c8689979-mhl7v
2026/09/07 11:21:26   11:17:40 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75c8689979 from 0 to 1
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: AddedInterface - Add eth0 [10.134.0.96/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:40 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Started - Started container manager
2026/09/07 11:21:26   11:17:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-sclh9: AddedInterface - Add eth0 [10.134.0.99/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Deployment/spark-operator-module-controller-manager: ScalingReplicaSet - Scaled up replica set spark-operator-module-controller-manager-59549b5854 from 0 to 1
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: AddedInterface - Add eth0 [10.134.0.97/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:40 INFO Pod/opendatahub-feast-operator-84ff599545-lnwt6: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:39 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-84glx to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:17:39 INFO Pod/ai-gateway-operator-89f69845b-lqv8q: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-lqv8q to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:39 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-dqh6n: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-dqh6n to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:39 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-84glx
2026/09/07 11:21:26   11:17:39 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-lqv8q
2026/09/07 11:21:26   11:17:39 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-dqh6n
2026/09/07 11:21:26   11:17:39 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/07 11:21:26   11:17:39 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/07 11:21:26   11:17:38 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Started - Started container manager
2026/09/07 11:21:26   11:17:38 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9_d744adfe-3717-44d9-8ebd-4d4044f27ad6 became leader
2026/09/07 11:21:26   11:17:38 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Created - Created container: manager
2026/09/07 11:21:26   11:17:38 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4" in 2.552s (2.552s including waiting). Image size: 143076452 bytes.
2026/09/07 11:21:26   11:17:37 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:37 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:36 INFO Pod/workbenches-operator-c674d9c9-p42hk: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" already present on machine
2026/09/07 11:21:26   11:17:36 INFO Pod/workbenches-operator-c674d9c9-p42hk: Started - Started container manager
2026/09/07 11:21:26   11:17:36 INFO Pod/workbenches-operator-c674d9c9-p42hk: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:35 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 1.079s (1.079s including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 1.088s (1.088s including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4"
2026/09/07 11:21:26   11:17:35 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Started - Started container manager
2026/09/07 11:21:26   11:17:35 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: AddedInterface - Add eth0 [10.134.0.94/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:35 INFO Pod/workbenches-operator-c674d9c9-p42hk: Pulled - Successfully pulled image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" in 2.177s (2.177s including waiting). Image size: 185267738 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" in 2.244s (2.244s including waiting). Image size: 198393268 bytes.
2026/09/07 11:21:26   11:17:35 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Started - Started container manager
2026/09/07 11:21:26   11:17:35 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Created - Created container: manager
2026/09/07 11:21:26   11:17:35 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Started - Started container manager
2026/09/07 11:21:26   11:17:34 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:34 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-6b668bbbb: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-6b668bbbbpv8l9
2026/09/07 11:21:26   11:17:34 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:34 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-6b668bbbb from 0 to 1
2026/09/07 11:21:26   11:17:34 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:34 WARN Pod/rhods-dashboard-bfcc58b46-42vcs: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:17:34 WARN Pod/rhods-dashboard-bfcc58b46-42vcs: Unhealthy - Readiness probe failed: Get "https://10.133.0.52:8943/healthcheck": dial tcp 10.133.0.52:8943: connect: connection refused
2026/09/07 11:21:26   11:17:33 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Created - Created container: manager
2026/09/07 11:21:26   11:17:33 INFO Pod/workbenches-operator-c674d9c9-p42hk: Pulling - Pulling image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898"
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 598ms (598ms including waiting). Image size: 207018016 bytes.
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 517ms (517ms including waiting). Image size: 209286205 bytes.
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 11:21:26   11:17:33 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Created - Created container: copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Started - Started container copy-manifests
2026/09/07 11:21:26   11:17:33 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Started - Started container manager
2026/09/07 11:21:26   11:17:32 INFO Pod/workbenches-operator-c674d9c9-p42hk: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-c674d9c9-p42hk to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-nhrpr to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-vfz9x to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-jvq7v to ip-10-0-128-124.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-2nlcj to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:32 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO Pod/workbenches-operator-c674d9c9-p42hk: AddedInterface - Add eth0 [10.134.0.93/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-c674d9c9 from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/workbenches-operator-c674d9c9: SuccessfulCreate - Created pod: workbenches-operator-c674d9c9-p42hk
2026/09/07 11:21:26   11:17:32 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-nhrpr
2026/09/07 11:21:26   11:17:32 INFO Pod/opendatahub-feast-operator-84ff599545-nhrpr: AddedInterface - Add eth0 [10.134.0.92/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 11:21:26   11:17:32 INFO Pod/ai-gateway-operator-89f69845b-2nlcj: AddedInterface - Add eth0 [10.134.0.90/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: AddedInterface - Add eth0 [10.134.0.91/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:32 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-vfz9x
2026/09/07 11:21:26   11:17:32 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-vfz9x: Pulling - Pulling image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2"
2026/09/07 11:21:26   11:17:32 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-2nlcj
2026/09/07 11:21:26   11:17:32 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-jvq7v
2026/09/07 11:21:26   11:17:32 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/07 11:21:26   11:17:32 INFO Pod/kserve-module-controller-manager-66ff7c977-jvq7v: AddedInterface - Add eth0 [10.132.0.83/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:30 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Created - Created container: manager
2026/09/07 11:21:26   11:17:30 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Pulled - Successfully pulled image "registry.k8s.io/pause:3.9" in 370ms (370ms including waiting). Image size: 750414 bytes.
2026/09/07 11:21:26   11:17:30 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Started - Started container manager
2026/09/07 11:21:26   11:17:29 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-6b8fb45c8b-hsr99 to ip-10-0-142-168.ec2.internal
2026/09/07 11:21:26   11:17:29 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-6b8fb45c8b from 0 to 1
2026/09/07 11:21:26   11:17:29 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: AddedInterface - Add eth0 [10.134.0.89/23] from ovn-kubernetes
2026/09/07 11:21:26   11:17:29 INFO ReplicaSet/feast-operator-controller-manager-6b8fb45c8b: SuccessfulCreate - Created pod: feast-operator-controller-manager-6b8fb45c8b-hsr99
2026/09/07 11:21:26   11:17:29 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-hsr99: Pulling - Pulling image "registry.k8s.io/pause:3.9"
2026/09/07 11:21:26   11:17:25 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:25 WARN Pod/rhods-dashboard-bfcc58b46-v2xm8: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:21:26   11:17:25 INFO Pod/dashboard-operator-669d6564bd-q49dz: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:25 INFO Pod/aihub-controller-manager-d877888b8-s4mh4: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:25 WARN Endpoints/aihub-controller-manager-metrics-service: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/aihub-controller-manager-metrics-service: Operation cannot be fulfilled on endpoints "aihub-controller-manager-metrics-service": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/aihub-controller-manager-metrics-service, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: a5481786-dca0-4fe8-b208-e35728ace0fb, UID in object meta: 
2026/09/07 11:21:26   11:17:24 INFO Pod/odh-observability-59db79cb46-jxh7c: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 INFO Pod/ai-gateway-operator-89f69845b-9gxl8: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 WARN Pod/kubeflow-trainer-controller-manager-5449979c85-h4tsl: Unhealthy - Readiness probe failed: Get "http://10.133.0.45:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
2026/09/07 11:21:26   11:17:24 INFO Pod/kserve-module-controller-manager-66ff7c977-8cc4s: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 WARN Pod/rhods-dashboard-bfcc58b46-v2xm8: Unhealthy - Readiness probe failed: Get "https://10.132.0.55:8943/healthcheck": dial tcp 10.132.0.55:8943: connect: connection refused
2026/09/07 11:21:26   11:17:24 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-2chm7: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:24 INFO Pod/workbenches-operator-c674d9c9-8qhfq: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:23 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:23 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:23 INFO Pod/opendatahub-feast-operator-84ff599545-w98d9: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:23 INFO Pod/mlflow-operator-controller-manager-5bbb468f5f-4cnxp: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/notebook-controller-deployment-75c8689979-4mdp7: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/odh-notebook-controller-manager-5bcb7b86db-24424: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/spark-operator-module-controller-manager-59549b5854-v7bd8: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:22 INFO Pod/opendatahub-ogx-operator-644d45d5b-cdxvd: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/odh-model-controller-6b9dd8b59d-gzhkt: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/spark-operator-controller-c594b967f-kgcc6: Killing - Stopping container controller
2026/09/07 11:21:26   11:17:21 INFO Pod/mlflow-ui-7bb6d46f5f-nkqj6: Killing - Stopping container mlflow-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-v2xm8: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-v2xm8: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-v2xm8: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:17:21 INFO Pod/model-registry-operator-controller-manager-6bcdfb5b87-ws2bb: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-42vcs: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-42vcs: Killing - Stopping container core-bff
2026/09/07 11:21:26   11:17:21 INFO Pod/rhods-dashboard-bfcc58b46-42vcs: Killing - Stopping container rhods-dashboard
2026/09/07 11:21:26   11:17:21 INFO Pod/notebooks-ui-6577cc467-vvp7j: Killing - Stopping container notebooks-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-kpjkz: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/dashboard-redirect-55f4444684-j8lnb: Killing - Stopping container nginx
2026/09/07 11:21:26   11:17:21 INFO Pod/model-registry-ui-67b8c946b9-s68np: Killing - Stopping container model-registry-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/kserve-controller-manager-5fbbd6dd88-zzwnk: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/gen-ai-ui-6459ffd979-sw7x5: Killing - Stopping container gen-ai-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/dashboard-redirect-55f4444684-htz29: Killing - Stopping container nginx
2026/09/07 11:21:26   11:17:21 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-f5p8j: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/automl-ui-74d9b97d49-x268c: Killing - Stopping container automl-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/autorag-ui-58f548cbd-jb9rf: Killing - Stopping container autorag-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/catalog-controller-manager-67d887945b-qlktz: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-pnjlp: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 INFO Pod/spark-operator-webhook-5d466b7d66-9vxzx: Killing - Stopping container webhook
2026/09/07 11:21:26   11:17:21 INFO Pod/maas-ui-675f94cfd8-b6vdb: Killing - Stopping container maas-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/model-serving-api-7df8c9655f-hbdt2: Killing - Stopping container server
2026/09/07 11:21:26   11:17:21 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-h4tsl: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 WARN Pod/maas-controller-56c898799-t7zrv: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "maas-controller-webhook-cert" not found
2026/09/07 11:21:26   11:17:21 INFO Pod/maas-controller-56c898799-t7zrv: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:21 WARN Pod/maas-controller-56c898799-t7zrv: FailedMount - MountVolume.SetUp failed for volume "metrics-certs" : secret "maas-controller-metrics-tls" not found
2026/09/07 11:21:26   11:17:21 INFO Pod/agent-ops-ui-7df989bc55-tcsxh: Killing - Stopping container agent-ops-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/eval-hub-ui-869b89d9b4-zsgms: Killing - Stopping container eval-hub-ui
2026/09/07 11:21:26   11:17:21 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8976vqd: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:20 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:20 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:20 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:20 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-8djfr: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:20 INFO Pod/kuberay-operator-7d87f54798-z9q95: Killing - Stopping container kuberay-operator
2026/09/07 11:21:26   11:17:20 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-x9blj: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:20 WARN Pod/maas-controller-56c898799-t7zrv: FailedToRetrieveImagePullSecret - Unable to retrieve some image pull secrets (maas-controller-dockercfg-vp7c6); attempting to pull the image may not succeed.
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:19 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:21:26   11:17:18 INFO Pod/kserve-localmodelnode-agent-4z77m: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:18 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-4z77m
2026/09/07 11:21:26   11:17:18 INFO Pod/kserve-localmodelnode-agent-c5dnt: Killing - Stopping container manager
2026/09/07 11:21:26   11:17:18 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-c5dnt
2026/09/07 11:21:26 === RESOURCE QUOTAS ===
2026/09/07 11:21:26 No resource quotas found in monitored namespaces
2026/09/07 11:21:26 Classification for Validate resource deletion recovery: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/07 11:21:26   Evidence: event Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get \"https://10.132.0.101:8444/healthz\": dial tcp 10.132.0.101:8444: connect: connection refused"],"confidence":"medium"}
2026/09/07 11:21:26 Diagnostics complete for failed test: Validate resource deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/07 11:24:42 TEST FAILURE DETECTED: group 1
2026/09/07 11:24:42 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 1) ===
2026/09/07 11:24:43 === CLUSTER STATE ===
2026/09/07 11:24:43 Node: ip-10-0-128-124.ec2.internal
2026/09/07 11:24:43   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:24:43   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:24:43   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:24:43   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:24:43   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 11:24:43 Node: ip-10-0-142-168.ec2.internal
2026/09/07 11:24:43   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:24:43   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:24:43   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:24:43   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:24:43   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:24:43 Node: ip-10-0-142-230.ec2.internal
2026/09/07 11:24:43   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:24:43   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:24:43   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:24:43   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:24:43   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:24:43 === DEPLOYMENTS ===
2026/09/07 11:24:43 Namespace: redhat-ods-applications
2026/09/07 11:24:43   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 11:24:43   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 11:24:43   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 11:24:43   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 11:24:43   Deployment maas-ui: 1/1 (READY)
2026/09/07 11:24:43   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 11:24:43   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 11:24:43 Namespace: kube-system
2026/09/07 11:24:43 === PODS ===
2026/09/07 11:24:43 Namespace redhat-ods-applications: no problematic pods found
2026/09/07 11:24:43 Namespace kube-system: no problematic pods found
2026/09/07 11:24:43 === OPERATOR STATUS ===
2026/09/07 11:24:43 Operator deployment: 3/3 ready
2026/09/07 11:24:43 === DSCI STATUS ===
2026/09/07 11:24:43 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 11:24:43 === DSC STATUS ===
2026/09/07 11:24:43 DSC default-dsc:
2026/09/07 11:24:43   Ready: True - 
2026/09/07 11:24:43   ProvisioningSucceeded: True - 
2026/09/07 11:24:43   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 11:24:43   ModulesReady: True - 
2026/09/07 11:24:43   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 11:24:43   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:24:43   DashboardReady: True - 
2026/09/07 11:24:43   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   KserveLLMInferenceServiceDependencies: False - Submodule ManagementState is set to Removed
2026/09/07 11:24:43   KserveLLMInferenceServiceWideEPDependencies: False - Submodule ManagementState is set to Removed
2026/09/07 11:24:43   KserveReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 11:24:43   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:24:43   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   RayReady: False - Component ManagementState is set to Removed
2026/09/07 11:24:43   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   TrainerReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 11:24:43   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 11:24:43   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 11:24:43 === RECENT EVENTS ===
2026/09/07 11:24:43   11:24:36 INFO Pod/kserve-module-controller-manager-66ff7c977-84glx: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:33 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:24:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:24:33 INFO Pod/kserve-localmodelnode-agent-c65mt: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:33 INFO Pod/odh-model-controller-57457c4c89-qkzqq: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:33 INFO Pod/model-serving-api-7df8c9655f-tfv8s: Killing - Stopping container server
2026/09/07 11:24:43   11:24:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:33 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-c65mt
2026/09/07 11:24:43   11:24:33 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:33 INFO Pod/kserve-localmodelnode-agent-4psfs: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:33 WARN Pod/odh-model-controller-57457c4c89-qkzqq: Unhealthy - Readiness probe failed: Get "http://10.132.0.107:8081/readyz": dial tcp 10.132.0.107:8081: connect: connection refused
2026/09/07 11:24:43   11:24:33 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:32 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-4psfs
2026/09/07 11:24:43   11:24:29 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-qkzqq_12fd3cc8-5f41-4522-b8d4-6592887d643b became leader
2026/09/07 11:24:43   11:24:12 INFO Pod/odh-model-controller-57457c4c89-qkzqq: Created - Created container: manager
2026/09/07 11:24:43   11:24:12 INFO Pod/odh-model-controller-57457c4c89-qkzqq: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 621ms (621ms including waiting). Image size: 239132625 bytes.
2026/09/07 11:24:43   11:24:12 INFO Pod/odh-model-controller-57457c4c89-qkzqq: Started - Started container manager
2026/09/07 11:24:43   11:24:11 INFO Pod/odh-model-controller-57457c4c89-qkzqq: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-qkzqq to ip-10-0-128-124.ec2.internal
2026/09/07 11:24:43   11:24:11 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 11:24:43   11:24:11 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-qkzqq
2026/09/07 11:24:43   11:24:11 INFO Pod/odh-model-controller-57457c4c89-qkzqq: AddedInterface - Add eth0 [10.132.0.107/23] from ovn-kubernetes
2026/09/07 11:24:43   11:24:11 INFO Pod/odh-model-controller-57457c4c89-qkzqq: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:24:43   11:24:07 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Killing - Stopping container manager
2026/09/07 11:24:43   11:24:07 INFO Pod/odh-model-controller-57457c4c89-8j6h9: Killing - Stopping container manager
2026/09/07 11:24:43   11:23:54 INFO Pod/model-serving-api-7df8c9655f-tfv8s: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-tfv8s to ip-10-0-128-124.ec2.internal
2026/09/07 11:24:43   11:23:54 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-tfv8s
2026/09/07 11:24:43   11:23:54 INFO Pod/model-serving-api-7df8c9655f-tfv8s: Started - Started container server
2026/09/07 11:24:43   11:23:54 INFO Pod/model-serving-api-7df8c9655f-tfv8s: AddedInterface - Add eth0 [10.132.0.106/23] from ovn-kubernetes
2026/09/07 11:24:43   11:23:54 INFO Pod/model-serving-api-7df8c9655f-tfv8s: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/07 11:24:43   11:23:54 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 11:24:43   11:23:54 INFO Pod/model-serving-api-7df8c9655f-tfv8s: Created - Created container: server
2026/09/07 11:24:43   11:23:52 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Killing - Stopping container server
2026/09/07 11:24:43   11:23:52 INFO Pod/model-serving-api-7df8c9655f-v8wbk: Killing - Stopping container server
2026/09/07 11:24:43   11:23:36 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-d67xd_05a09630-bd97-40db-a2c0-6f699bcc686a became leader
2026/09/07 11:24:43   11:23:18 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: Started - Started container manager
2026/09/07 11:24:43   11:23:18 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 429ms (429ms including waiting). Image size: 235279660 bytes.
2026/09/07 11:24:43   11:23:18 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: Created - Created container: manager
2026/09/07 11:24:43   11:23:17 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-d67xd to ip-10-0-142-168.ec2.internal
2026/09/07 11:24:43   11:23:17 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: AddedInterface - Add eth0 [10.134.0.113/23] from ovn-kubernetes
2026/09/07 11:24:43   11:23:17 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 11:24:43   11:23:17 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cszzw: Killing - Stopping container manager
2026/09/07 11:24:43   11:23:17 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-d67xd: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:24:43   11:23:17 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Killing - Stopping container manager
2026/09/07 11:24:43   11:23:17 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-d67xd
2026/09/07 11:24:43   11:22:37 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 478ms (478ms including waiting). Image size: 244249302 bytes.
2026/09/07 11:24:43   11:22:37 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: Created - Created container: manager
2026/09/07 11:24:43   11:22:37 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: Started - Started container manager
2026/09/07 11:24:43   11:22:36 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: AddedInterface - Add eth0 [10.132.0.105/23] from ovn-kubernetes
2026/09/07 11:24:43   11:22:36 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:24:43   11:22:35 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-z97bj: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-z97bj to ip-10-0-128-124.ec2.internal
2026/09/07 11:24:43   11:22:35 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 11:24:43   11:22:35 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-z97bj
2026/09/07 11:24:43   11:22:31 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rgvs7: Killing - Stopping container manager
2026/09/07 11:24:43   11:22:31 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4lrwg: Killing - Stopping container manager
2026/09/07 11:24:43   11:22:18 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-lsvw7_a8fbf454-54f5-43cc-8cf9-4420035835bd became leader
2026/09/07 11:24:43   11:22:00 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: Started - Started container manager
2026/09/07 11:24:43   11:22:00 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 372ms (372ms including waiting). Image size: 247667445 bytes.
2026/09/07 11:24:43   11:22:00 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: Created - Created container: manager
2026/09/07 11:24:43   11:21:59 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-lsvw7 to ip-10-0-142-230.ec2.internal
2026/09/07 11:24:43   11:21:59 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-lsvw7
2026/09/07 11:24:43   11:21:59 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 11:24:43   11:21:59 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:24:43   11:21:59 INFO Pod/kserve-controller-manager-5fbbd6dd88-lsvw7: AddedInterface - Add eth0 [10.133.0.117/23] from ovn-kubernetes
2026/09/07 11:24:43   11:21:56 INFO Pod/kserve-controller-manager-5fbbd6dd88-wrr9b: Killing - Stopping container manager
2026/09/07 11:24:43   11:21:56 INFO Pod/kserve-controller-manager-5fbbd6dd88-v76tn: Killing - Stopping container manager
2026/09/07 11:24:43   11:21:43 WARN Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
2026/09/07 11:24:43   11:21:33 WARN Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:24:43   11:21:33 WARN Pod/rhods-dashboard-7f94fcf99d-6cjmj: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8943/healthcheck": dial tcp 10.132.0.101:8943: connect: connection refused
2026/09/07 11:24:43   11:21:16 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:21:16 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container rhods-dashboard
2026/09/07 11:24:43   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container core-bff
2026/09/07 11:24:43   11:21:16 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7f94fcf99d from 1 to 0
2026/09/07 11:24:43   11:21:16 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulDelete - Deleted pod: rhods-dashboard-7f94fcf99d-6cjmj
2026/09/07 11:24:43   11:21:16 INFO Pod/rhods-dashboard-7f94fcf99d-6cjmj: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:24:43   11:21:11 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:21:11 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:21:05 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe failed: Get "https://10.133.0.111:8444/healthz": dial tcp 10.133.0.111:8444: connect: connection refused
2026/09/07 11:24:43   11:20:55 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe failed: Get "https://10.133.0.111:8943/healthcheck": dial tcp 10.133.0.111:8943: connect: connection refused
2026/09/07 11:24:43   11:20:55 WARN Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:24:43   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 388ms (388ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:24:43   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container core-bff
2026/09/07 11:24:43   11:20:38 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: core-bff
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container kube-rbac-proxy
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Started - Started container rhods-dashboard
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: kube-rbac-proxy
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Created - Created container: rhods-dashboard
2026/09/07 11:24:43   11:20:37 INFO Pod/rhods-dashboard-775867476c-f7trw: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 594ms (594ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:24:43   11:20:36 INFO Pod/rhods-dashboard-775867476c-f7trw: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-f7trw to ip-10-0-142-230.ec2.internal
2026/09/07 11:24:43   11:20:36 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-f7trw
2026/09/07 11:24:43   11:20:36 INFO ReplicaSet/rhods-dashboard-5c9d44b7b4: SuccessfulDelete - Deleted pod: rhods-dashboard-5c9d44b7b4-dgpl5
2026/09/07 11:24:43   11:20:36 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-5c9d44b7b4 from 1 to 0
2026/09/07 11:24:43   11:20:36 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-775867476c from 1 to 2
2026/09/07 11:24:43   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container rhods-dashboard
2026/09/07 11:24:43   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container core-bff
2026/09/07 11:24:43   11:20:36 INFO Pod/rhods-dashboard-5c9d44b7b4-dgpl5: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:24:43   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 2.22s (2.22s including waiting). Image size: 239132625 bytes.
2026/09/07 11:24:43   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Created - Created container: manager
2026/09/07 11:24:43   11:20:33 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Started - Started container manager
2026/09/07 11:24:43   11:20:31 INFO Pod/odh-model-controller-57457c4c89-bgwgw: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/07 11:24:43   11:20:31 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:24:43   11:20:30 INFO Pod/odh-model-controller-57457c4c89-bgwgw: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-bgwgw to ip-10-0-142-230.ec2.internal
2026/09/07 11:24:43   11:20:30 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-bgwgw
2026/09/07 11:24:43   11:20:30 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/07 11:24:43   11:20:27 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:24:43   11:20:27 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe failed: Get "https://10.133.0.114:8943/healthcheck": dial tcp 10.133.0.114:8943: connect: connection refused
2026/09/07 11:24:43   11:20:23 WARN Pod/rhods-dashboard-7f94fcf99d-rwzbj: Unhealthy - Readiness probe failed: Get "https://10.133.0.114:8444/healthz": dial tcp 10.133.0.114:8444: connect: connection refused
2026/09/07 11:24:43   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Created - Created container: server
2026/09/07 11:24:43   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Started - Started container server
2026/09/07 11:24:43   11:20:12 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.284s (1.284s including waiting). Image size: 172459788 bytes.
2026/09/07 11:24:43   11:20:11 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 11:24:43   11:20:11 INFO Pod/model-serving-api-7df8c9655f-dz7bh: AddedInterface - Add eth0 [10.134.0.112/23] from ovn-kubernetes
2026/09/07 11:24:43   11:20:10 INFO Pod/model-serving-api-7df8c9655f-dz7bh: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-dz7bh to ip-10-0-142-168.ec2.internal
2026/09/07 11:24:43   11:20:10 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-dz7bh
2026/09/07 11:24:43   11:20:10 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/07 11:24:43   11:20:01 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe failed: Get "https://10.134.0.106:8444/healthz": dial tcp 10.134.0.106:8444: connect: connection refused
2026/09/07 11:24:43   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 373ms (373ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:24:43   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: core-bff
2026/09/07 11:24:43   11:19:59 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container core-bff
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: kube-rbac-proxy
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: AddedInterface - Add eth0 [10.134.0.111/23] from ovn-kubernetes
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 429ms (429ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container kube-rbac-proxy
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Started - Started container rhods-dashboard
2026/09/07 11:24:43   11:19:58 INFO Pod/rhods-dashboard-775867476c-bpspj: Created - Created container: rhods-dashboard
2026/09/07 11:24:43   11:19:57 INFO Pod/rhods-dashboard-775867476c-bpspj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-bpspj to ip-10-0-142-168.ec2.internal
2026/09/07 11:24:43   11:19:57 WARN Endpoints/odh-dashboard-automl-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-automl-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-automl-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-automl-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 9931d5c4-97af-4cd3-9b95-ee5288a56884, UID in object meta: 
2026/09/07 11:24:43   11:19:57 WARN Endpoints/odh-dashboard-autorag-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-autorag-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-autorag-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-autorag-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 79bb98aa-3dac-4c9d-ae1e-46c4f3573e85, UID in object meta: 
2026/09/07 11:24:43   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container rhods-dashboard
2026/09/07 11:24:43   11:19:57 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-bpspj
2026/09/07 11:24:43   11:19:57 INFO Pod/autorag-ui-58f548cbd-8pn4g: Killing - Stopping container autorag-ui
2026/09/07 11:24:43   11:19:57 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-775867476c from 0 to 1
2026/09/07 11:24:43   11:19:57 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7f94fcf99d from 2 to 1
2026/09/07 11:24:43   11:19:57 INFO ReplicaSet/rhods-dashboard-7f94fcf99d: SuccessfulDelete - Deleted pod: rhods-dashboard-7f94fcf99d-rwzbj
2026/09/07 11:24:43   11:19:57 INFO Pod/automl-ui-74d9b97d49-lzkxj: Killing - Stopping container automl-ui
2026/09/07 11:24:43   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container core-bff
2026/09/07 11:24:43   11:19:57 INFO Pod/rhods-dashboard-7f94fcf99d-rwzbj: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:24:43   11:19:56 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe failed: Get "https://10.134.0.106:8943/healthcheck": dial tcp 10.134.0.106:8943: connect: connection refused
2026/09/07 11:24:43   11:19:56 WARN Pod/rhods-dashboard-5c9d44b7b4-zhmtt: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:24:43   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 1.941s (1.941s including waiting). Image size: 235279660 bytes.
2026/09/07 11:24:43   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Created - Created container: manager
2026/09/07 11:24:43   11:19:53 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Started - Started container manager
2026/09/07 11:24:43   11:19:51 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:24:43   11:19:51 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: AddedInterface - Add eth0 [10.132.0.104/23] from ovn-kubernetes
2026/09/07 11:24:43   11:19:50 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-hvhs7: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-hvhs7 to ip-10-0-128-124.ec2.internal
2026/09/07 11:24:43   11:19:50 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-hvhs7
2026/09/07 11:24:43   11:19:50 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/07 11:24:43   11:19:46 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:19:46 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:24:43   11:19:46 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f897kgkm: Killing - Stopping container manager
2026/09/07 11:24:43 === RESOURCE QUOTAS ===
2026/09/07 11:24:43 No resource quotas found in monitored namespaces
2026/09/07 11:24:43 Classification for group 1: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/07 11:24:43   Evidence: event Pod/odh-model-controller-57457c4c89-qkzqq: Unhealthy - Readiness probe failed: Get "http://10.132.0.107:8081/readyz": dial tcp 10.132.0.107:8081: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/odh-model-controller-57457c4c89-qkzqq: Unhealthy - Readiness probe failed: Get \"http://10.132.0.107:8081/readyz\": dial tcp 10.132.0.107:8081: connect: connection refused"],"confidence":"medium"}
2026/09/07 11:24:43 Diagnostics complete for failed test: group 1
    controller_test.go:762: Stopping: group 1 test failed.
2026/09/07 11:24:43 Circuit breaker: failure classified as infrastructure/probe-failure (code=1014)
2026/09/07 11:24:43 Circuit breaker: infrastructure/unknown failure recorded (1/3)
    helper_test.go:156: Verifying no Deployments remain in namespace redhat-ods-applications with part-of=kueue label.
    helper_test.go:156: Ensuring the component CR /default-kueue is removed.
2026-09-07T11:31:10Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
2026/09/07 11:34:22 TEST FAILURE DETECTED: components
2026/09/07 11:34:22 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:components) ===
2026/09/07 11:34:22 === CLUSTER STATE ===
2026/09/07 11:34:22 Node: ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:34:22   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:34:22   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:34:22   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:34:22   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 11:34:22 Node: ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:34:22   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:34:22   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:34:22   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:34:22   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:34:22 Node: ip-10-0-142-230.ec2.internal
2026/09/07 11:34:22   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 11:34:22   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 11:34:22   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 11:34:22   Condition Ready: True - kubelet is posting ready status
2026/09/07 11:34:22   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/07 11:34:22 === DEPLOYMENTS ===
2026/09/07 11:34:22 Namespace: redhat-ods-applications
2026/09/07 11:34:22   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 11:34:22   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 11:34:22   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 11:34:22   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 11:34:22   Deployment kserve-controller-manager: 1/1 (READY)
2026/09/07 11:34:22   Deployment kserve-localmodel-controller-manager: 1/1 (READY)
2026/09/07 11:34:22   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/07 11:34:22   Deployment kubeflow-trainer-controller-manager: 1/1 (READY)
2026/09/07 11:34:22   Deployment llmisvc-controller-manager: 1/1 (READY)
2026/09/07 11:34:22   Deployment maas-ui: 1/1 (READY)
2026/09/07 11:34:22   Deployment model-serving-api: 1/1 (READY)
2026/09/07 11:34:22   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 11:34:22   Deployment odh-model-controller: 1/1 (READY)
2026/09/07 11:34:22   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 11:34:22   Deployment trainer-operator-controller-manager: 1/1 (READY)
2026/09/07 11:34:22 Namespace: kube-system
2026/09/07 11:34:22 === PODS ===
2026/09/07 11:34:22 Namespace redhat-ods-applications: no problematic pods found
2026/09/07 11:34:22 Namespace kube-system: no problematic pods found
2026/09/07 11:34:22 === OPERATOR STATUS ===
2026/09/07 11:34:22 Operator deployment: 3/3 ready
2026/09/07 11:34:22 === DSCI STATUS ===
2026/09/07 11:34:22 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 11:34:22 === DSC STATUS ===
2026/09/07 11:34:22 DSC default-dsc:
2026/09/07 11:34:22   Ready: True - 
2026/09/07 11:34:22   ProvisioningSucceeded: True - 
2026/09/07 11:34:22   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 11:34:22   ModulesReady: True - 
2026/09/07 11:34:22   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 11:34:22   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:34:22   DashboardReady: True - 
2026/09/07 11:34:22   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   KserveLLMInferenceServiceDependencies: True - 
2026/09/07 11:34:22   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/07 11:34:22   KserveReady: True - 
2026/09/07 11:34:22   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 11:34:22   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 11:34:22   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   RayReady: False - Component ManagementState is set to Removed
2026/09/07 11:34:22   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   TrainerReady: True - 
2026/09/07 11:34:22   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 11:34:22   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 11:34:22   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 11:34:22 === RECENT EVENTS ===
2026/09/07 11:34:22   11:34:01 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-4dmsk_f4ad3931-9a82-4ba6-8cc0-a940b2842422 became leader
2026/09/07 11:34:22   11:34:01 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-pwbvv_7aab70c0-23e0-4804-ad91-b6ec02fff89a became leader
2026/09/07 11:34:22   11:33:59 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-xt6c2_d55d6f85-e771-4ae0-b9ec-0498810a66eb became leader
2026/09/07 11:34:22   11:33:48 INFO Pod/kserve-localmodelnode-agent-zfds9: Started - Started container manager
2026/09/07 11:34:22   11:33:48 INFO Pod/kserve-localmodelnode-agent-zfds9: Created - Created container: manager
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-vf2zp: Started - Started container manager
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-zfds9: AddedInterface - Add eth0 [10.134.0.139/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-zfds9: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-zfds9: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 789ms (789ms including waiting). Image size: 244000959 bytes.
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-vf2zp: Created - Created container: manager
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-vf2zp: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 389ms (389ms including waiting). Image size: 244000959 bytes.
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-vf2zp: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:34:22   11:33:47 INFO Pod/kserve-localmodelnode-agent-vf2zp: AddedInterface - Add eth0 [10.132.0.131/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:46 INFO Pod/kserve-localmodelnode-agent-zfds9: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-zfds9 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:33:46 INFO Pod/kserve-localmodelnode-agent-vf2zp: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-vf2zp to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:33:43 INFO Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: Created - Created container: manager
2026/09/07 11:34:22   11:33:43 INFO Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: Started - Started container manager
2026/09/07 11:34:22   11:33:43 INFO Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 443ms (443ms including waiting). Image size: 247667445 bytes.
2026/09/07 11:34:22   11:33:42 WARN Pod/kserve-localmodelnode-agent-zfds9: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 11:34:22   11:33:42 WARN Pod/kserve-localmodelnode-agent-vf2zp: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 11:34:22   11:33:42 INFO Pod/odh-model-controller-57457c4c89-4dmsk: AddedInterface - Add eth0 [10.132.0.130/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-pwbvv: Created - Created container: manager
2026/09/07 11:34:22   11:33:42 INFO Pod/model-serving-api-7df8c9655f-7rjb7: Started - Started container server
2026/09/07 11:34:22   11:33:42 INFO Pod/model-serving-api-7df8c9655f-7rjb7: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/07 11:34:22   11:33:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-pwbvv: AddedInterface - Add eth0 [10.134.0.137/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-pwbvv: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:34:22   11:33:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-pwbvv: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 553ms (553ms including waiting). Image size: 235279660 bytes.
2026/09/07 11:34:22   11:33:42 INFO Pod/odh-model-controller-57457c4c89-4dmsk: Started - Started container manager
2026/09/07 11:34:22   11:33:42 INFO Pod/odh-model-controller-57457c4c89-4dmsk: Created - Created container: manager
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: AddedInterface - Add eth0 [10.134.0.136/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-djt7x: AddedInterface - Add eth0 [10.132.0.129/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-djt7x: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-djt7x: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 503ms (503ms including waiting). Image size: 244249302 bytes.
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-djt7x: Created - Created container: manager
2026/09/07 11:34:22   11:33:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-djt7x: Started - Started container manager
2026/09/07 11:34:22   11:33:42 INFO Pod/odh-model-controller-57457c4c89-4dmsk: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 442ms (442ms including waiting). Image size: 239132625 bytes.
2026/09/07 11:34:22   11:33:42 INFO Pod/odh-model-controller-57457c4c89-4dmsk: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:34:22   11:33:42 INFO Pod/model-serving-api-7df8c9655f-7rjb7: AddedInterface - Add eth0 [10.134.0.138/23] from ovn-kubernetes
2026/09/07 11:34:22   11:33:42 INFO Pod/model-serving-api-7df8c9655f-7rjb7: Created - Created container: server
2026/09/07 11:34:22   11:33:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-vf2zp
2026/09/07 11:34:22   11:33:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-zfds9
2026/09/07 11:34:22   11:33:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-pwbvv: Started - Started container manager
2026/09/07 11:34:22   11:33:41 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:33:41 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:33:41 INFO Pod/odh-model-controller-57457c4c89-4dmsk: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-4dmsk to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:33:41 INFO Pod/model-serving-api-7df8c9655f-7rjb7: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-7rjb7 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:33:41 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-pwbvv: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-pwbvv to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:33:41 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-djt7x: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-djt7x to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:33:41 INFO Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-xt6c2 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:33:41 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-djt7x
2026/09/07 11:34:22   11:33:41 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2":
 Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:34:22   11:33:41 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 11:34:22   11:33:41 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 11:34:22   11:33:41 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 11:34:22   11:33:41 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 11:34:22   11:33:41 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "openshift-ai-inferenceservice-image-volume-scc": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "openshift-ai-llminferenceservice-scc": Forbidden: not usable by user or serviceaccount, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable
 by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:34:22   11:33:41 WARN Pod/kserve-controller-manager-5fbbd6dd88-xt6c2: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kserve-webhook-server-cert" not found
2026/09/07 11:34:22   11:33:41 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-pwbvv
2026/09/07 11:34:22   11:33:41 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-4dmsk
2026/09/07 11:34:22   11:33:41 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 11:34:22   11:33:41 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-xt6c2
2026/09/07 11:34:22   11:33:41 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-7rjb7
2026/09/07 11:34:22   11:33:14 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-l8vxt_bb56c726-46a8-49e4-8f43-c89ec79e7485 became leader
2026/09/07 11:34:22   11:32:55 WARN Pod/rhods-dashboard-7495c45f5c-ssd86: Unhealthy - Readiness probe failed: Get "https://10.132.0.123:8444/healthz": dial tcp 10.132.0.123:8444: connect: connection refused
2026/09/07 11:34:22   11:32:54 INFO Pod/kserve-module-controller-manager-66ff7c977-l8vxt: Created - Created container: manager
2026/09/07 11:34:22   11:32:54 INFO Pod/kserve-module-controller-manager-66ff7c977-l8vxt: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/07 11:34:22   11:32:54 INFO Pod/kserve-module-controller-manager-66ff7c977-l8vxt: Started - Started container manager
2026/09/07 11:34:22   11:32:53 INFO Pod/kserve-module-controller-manager-66ff7c977-l8vxt: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-l8vxt to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:32:53 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 11:34:22   11:32:53 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-l8vxt
2026/09/07 11:34:22   11:32:53 INFO Pod/kserve-module-controller-manager-66ff7c977-l8vxt: AddedInterface - Add eth0 [10.134.0.135/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:44 WARN Pod/kserve-module-controller-manager-66ff7c977-4fxv8: Unhealthy - Readiness probe failed: Get "http://10.134.0.129:8081/readyz": dial tcp 10.134.0.129:8081: connect: connection refused
2026/09/07 11:34:22   11:32:44 INFO Pod/kserve-module-controller-manager-66ff7c977-4fxv8: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:32:43 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:32:43 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-8z62g
2026/09/07 11:34:22   11:32:43 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 INFO Pod/kserve-localmodelnode-agent-t6lz7: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 INFO Pod/odh-model-controller-57457c4c89-bjb7h: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 INFO Pod/model-serving-api-7df8c9655f-n89d9: Killing - Stopping container server
2026/09/07 11:34:22   11:32:43 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-t6lz7
2026/09/07 11:34:22   11:32:43 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 INFO Pod/kserve-localmodelnode-agent-8z62g: Killing - Stopping container manager
2026/09/07 11:34:22   11:32:43 WARN Pod/model-serving-api-7df8c9655f-n89d9: Unhealthy - Readiness probe failed: Get "https://10.134.0.133:8443/readyz": dial tcp 10.134.0.133:8443: connect: connection refused
2026/09/07 11:34:22   11:32:35 WARN Pod/rhods-dashboard-7495c45f5c-ssd86: Unhealthy - Readiness probe failed: Get "https://10.132.0.123:8943/healthcheck": dial tcp 10.132.0.123:8943: connect: connection refused
2026/09/07 11:34:22   11:32:35 WARN Pod/rhods-dashboard-7495c45f5c-ssd86: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:34:22   11:32:25 INFO Pod/rhods-dashboard-7495c45f5c-ssd86: Killing - Stopping container rhods-dashboard
2026/09/07 11:34:22   11:32:25 INFO Pod/rhods-dashboard-7495c45f5c-ssd86: Killing - Stopping container core-bff
2026/09/07 11:34:22   11:32:25 INFO Pod/rhods-dashboard-7495c45f5c-ssd86: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:34:22   11:32:25 INFO ReplicaSet/rhods-dashboard-7495c45f5c: SuccessfulDelete - Deleted pod: rhods-dashboard-7495c45f5c-ssd86
2026/09/07 11:34:22   11:32:22 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-rl5lv_8cf3e610-b547-4ee5-85c7-8abe52077a15 became leader
2026/09/07 11:34:22   11:32:20 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-qqbls_e31df76b-3ab6-4a98-8a53-edd8692f5cb1 became leader
2026/09/07 11:34:22   11:32:19 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-bjb7h_6f74577f-977f-4227-8648-5ce046a22ec2 became leader
2026/09/07 11:34:22   11:32:18 INFO Pod/kserve-localmodelnode-agent-8z62g: Created - Created container: manager
2026/09/07 11:34:22   11:32:18 INFO Pod/kserve-localmodelnode-agent-8z62g: Started - Started container manager
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-8z62g: AddedInterface - Add eth0 [10.132.0.128/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-8z62g: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-8z62g: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 470ms (470ms including waiting). Image size: 244000959 bytes.
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-t6lz7: AddedInterface - Add eth0 [10.134.0.134/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-t6lz7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-t6lz7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 610ms (610ms including waiting). Image size: 244000959 bytes.
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-t6lz7: Created - Created container: manager
2026/09/07 11:34:22   11:32:17 INFO Pod/kserve-localmodelnode-agent-t6lz7: Started - Started container manager
2026/09/07 11:34:22   11:32:16 INFO Pod/kserve-localmodelnode-agent-8z62g: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-8z62g to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:32:16 INFO Pod/kserve-localmodelnode-agent-t6lz7: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-t6lz7 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:32:16 WARN Pod/kserve-localmodelnode-agent-8z62g: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 11:34:22   11:32:13 WARN Pod/rhods-dashboard-7495c45f5c-dt7qv: Unhealthy - Readiness probe failed: Get "https://10.134.0.125:8943/healthcheck": dial tcp 10.134.0.125:8943: connect: connection refused
2026/09/07 11:34:22   11:32:13 WARN Pod/rhods-dashboard-7495c45f5c-dt7qv: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:34:22   11:32:13 WARN Pod/rhods-dashboard-7495c45f5c-dt7qv: Unhealthy - Readiness probe failed: Get "https://10.134.0.125:8444/healthz": dial tcp 10.134.0.125:8444: connect: connection refused
2026/09/07 11:34:22   11:32:03 INFO Pod/odh-model-controller-57457c4c89-bjb7h: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 412ms (412ms including waiting). Image size: 239132625 bytes.
2026/09/07 11:34:22   11:32:03 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: Started - Started container manager
2026/09/07 11:34:22   11:32:03 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 491ms (491ms including waiting). Image size: 247667445 bytes.
2026/09/07 11:34:22   11:32:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 11:34:22   11:32:03 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: Started - Started container manager
2026/09/07 11:34:22   11:32:03 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: Created - Created container: manager
2026/09/07 11:34:22   11:32:03 INFO Pod/odh-model-controller-57457c4c89-bjb7h: AddedInterface - Add eth0 [10.133.0.124/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:03 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 558ms (558ms including waiting). Image size: 244249302 bytes.
2026/09/07 11:34:22   11:32:03 INFO Pod/odh-model-controller-57457c4c89-bjb7h: Started - Started container manager
2026/09/07 11:34:22   11:32:03 INFO Pod/odh-model-controller-57457c4c89-bjb7h: Created - Created container: manager
2026/09/07 11:34:22   11:32:03 INFO Pod/odh-model-controller-57457c4c89-bjb7h: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 11:34:22   11:32:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: Started - Started container manager
2026/09/07 11:34:22   11:32:03 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: Created - Created container: manager
2026/09/07 11:34:22   11:32:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 477ms (477ms including waiting). Image size: 235279660 bytes.
2026/09/07 11:34:22   11:32:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: AddedInterface - Add eth0 [10.134.0.132/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: Created - Created container: manager
2026/09/07 11:34:22   11:32:02 WARN Pod/kserve-localmodelnode-agent-t6lz7: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 11:34:22   11:32:02 WARN Pod/kserve-localmodelnode-agent-8z62g: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 11:34:22   11:32:02 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:32:02 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:32:02 INFO Pod/odh-model-controller-57457c4c89-bjb7h: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-bjb7h to ip-10-0-142-230.ec2.internal
2026/09/07 11:34:22   11:32:02 INFO Pod/model-serving-api-7df8c9655f-n89d9: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-n89d9 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:32:02 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-rl5lv: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-rl5lv to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:32:02 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-cjqlb to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:32:02 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-qqbls to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:32:02 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: AddedInterface - Add eth0 [10.132.0.127/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:02 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: AddedInterface - Add eth0 [10.134.0.131/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:02 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-t6lz7
2026/09/07 11:34:22   11:32:02 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-8z62g
2026/09/07 11:34:22   11:32:02 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "openshift-ai-inferenceservice-image-volume-scc": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "openshift-ai-llminferenceservice-scc": Forbidden: not usable by user or serviceaccount, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable
 by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:34:22   11:32:02 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2":
 Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 11:34:22   11:32:02 INFO Pod/kserve-controller-manager-5fbbd6dd88-qqbls: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 11:34:22   11:32:02 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-qqbls
2026/09/07 11:34:22   11:32:02 INFO Pod/model-serving-api-7df8c9655f-n89d9: AddedInterface - Add eth0 [10.134.0.133/23] from ovn-kubernetes
2026/09/07 11:34:22   11:32:02 INFO Pod/model-serving-api-7df8c9655f-n89d9: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/07 11:34:22   11:32:02 INFO Pod/model-serving-api-7df8c9655f-n89d9: Created - Created container: server
2026/09/07 11:34:22   11:32:02 INFO Pod/model-serving-api-7df8c9655f-n89d9: Started - Started container server
2026/09/07 11:34:22   11:32:02 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 11:34:22   11:32:02 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 11:34:22   11:32:02 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cjqlb: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 11:34:22   11:32:02 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-n89d9
2026/09/07 11:34:22   11:32:02 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-bjb7h
2026/09/07 11:34:22   11:32:02 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 11:34:22   11:32:02 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-rl5lv
2026/09/07 11:34:22   11:32:02 WARN Pod/odh-model-controller-57457c4c89-bjb7h: FailedMount - MountVolume.SetUp failed for volume "metrics-certs" : secret "odh-model-controller-metrics-tls" not found
2026/09/07 11:34:22   11:32:02 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 11:34:22   11:32:02 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-cjqlb
2026/09/07 11:34:22   11:32:02 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 11:34:22   11:31:48 INFO Pod/rhods-dashboard-775867476c-pg5h8: Created - Created container: kube-rbac-proxy
2026/09/07 11:34:22   11:31:48 INFO Pod/rhods-dashboard-775867476c-pg5h8: Created - Created container: core-bff
2026/09/07 11:34:22   11:31:48 INFO Pod/rhods-dashboard-775867476c-pg5h8: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 406ms (406ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:34:22   11:31:48 INFO Pod/rhods-dashboard-775867476c-pg5h8: Started - Started container core-bff
2026/09/07 11:34:22   11:31:48 INFO Pod/rhods-dashboard-775867476c-pg5h8: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:34:22   11:31:48 INFO Pod/rhods-dashboard-775867476c-pg5h8: Started - Started container kube-rbac-proxy
2026/09/07 11:34:22   11:31:47 INFO Pod/rhods-dashboard-775867476c-pg5h8: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:34:22   11:31:47 INFO Pod/rhods-dashboard-775867476c-pg5h8: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:34:22   11:31:47 INFO Pod/rhods-dashboard-775867476c-pg5h8: Started - Started container rhods-dashboard
2026/09/07 11:34:22   11:31:47 INFO Pod/rhods-dashboard-775867476c-pg5h8: Created - Created container: rhods-dashboard
2026/09/07 11:34:22   11:31:47 INFO Pod/rhods-dashboard-775867476c-pg5h8: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 371ms (371ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:34:22   11:31:47 INFO Pod/rhods-dashboard-775867476c-pg5h8: AddedInterface - Add eth0 [10.132.0.126/23] from ovn-kubernetes
2026/09/07 11:34:22   11:31:46 INFO Pod/rhods-dashboard-775867476c-pg5h8: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-pg5h8 to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:31:46 INFO ReplicaSet/rhods-dashboard-7495c45f5c: SuccessfulDelete - Deleted pod: rhods-dashboard-7495c45f5c-dt7qv
2026/09/07 11:34:22   11:31:46 INFO Pod/rhods-dashboard-7495c45f5c-dt7qv: Killing - Stopping container core-bff
2026/09/07 11:34:22   11:31:46 INFO Pod/rhods-dashboard-7495c45f5c-dt7qv: Killing - Stopping container kube-rbac-proxy
2026/09/07 11:34:22   11:31:46 INFO Pod/rhods-dashboard-7495c45f5c-dt7qv: Killing - Stopping container rhods-dashboard
2026/09/07 11:34:22   11:31:46 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-pg5h8
2026/09/07 11:34:22   11:31:46 INFO Deployment/rhods-dashboard: ScalingReplicaSet - (combined from similar events): Scaled down replica set rhods-dashboard-7495c45f5c from 2 to 1
2026/09/07 11:34:22   11:31:36 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-cjd4x_7a017a91-ca97-4580-a1d0-a92bdadd674d became leader
2026/09/07 11:34:22   11:31:34 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-4fxv8_aec69b6e-dfef-4a43-a1fb-662c9183bea1 became leader
2026/09/07 11:34:22   11:31:19 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-cjd4x: Started - Started container manager
2026/09/07 11:34:22   11:31:19 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-cjd4x: Created - Created container: manager
2026/09/07 11:34:22   11:31:19 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-cjd4x: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" in 1.336s (1.336s including waiting). Image size: 158342259 bytes.
2026/09/07 11:34:22   11:31:18 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-cjd4x: AddedInterface - Add eth0 [10.132.0.125/23] from ovn-kubernetes
2026/09/07 11:34:22   11:31:18 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-cjd4x: Pulling - Pulling image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35"
2026/09/07 11:34:22   11:31:17 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:31:17 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-cjd4x: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-cjd4x to ip-10-0-128-124.ec2.internal
2026/09/07 11:34:22   11:31:17 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/07 11:34:22   11:31:17 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-cjd4x
2026/09/07 11:34:22   11:31:13 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-jsjbq: Started - Started container manager
2026/09/07 11:34:22   11:31:13 INFO Pod/kserve-module-controller-manager-66ff7c977-4fxv8: Started - Started container manager
2026/09/07 11:34:22   11:31:13 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-jsjbq: Pulled - Container image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" already present on machine
2026/09/07 11:34:22   11:31:13 INFO Pod/kserve-module-controller-manager-66ff7c977-4fxv8: AddedInterface - Add eth0 [10.134.0.129/23] from ovn-kubernetes
2026/09/07 11:34:22   11:31:13 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-jsjbq: Created - Created container: manager
2026/09/07 11:34:22   11:31:13 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-jsjbq: AddedInterface - Add eth0 [10.134.0.130/23] from ovn-kubernetes
2026/09/07 11:34:22   11:31:13 INFO Pod/kserve-module-controller-manager-66ff7c977-4fxv8: Created - Created container: manager
2026/09/07 11:34:22   11:31:13 INFO Pod/kserve-module-controller-manager-66ff7c977-4fxv8: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/07 11:34:22   11:31:12 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-jsjbq: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-jsjbq to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:31:12 INFO Pod/kserve-module-controller-manager-66ff7c977-4fxv8: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-4fxv8 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:31:12 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-jsjbq
2026/09/07 11:34:22   11:31:12 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/07 11:34:22   11:31:12 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-4fxv8
2026/09/07 11:34:22   11:31:12 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 11:34:22   11:31:09 INFO Pod/rhods-dashboard-775867476c-6thcb: Started - Started container core-bff
2026/09/07 11:34:22   11:31:09 INFO Pod/rhods-dashboard-775867476c-6thcb: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 474ms (474ms including waiting). Image size: 181738206 bytes.
2026/09/07 11:34:22   11:31:09 INFO Pod/rhods-dashboard-775867476c-6thcb: Created - Created container: core-bff
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Created - Created container: kube-rbac-proxy
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Started - Started container kube-rbac-proxy
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Started - Started container rhods-dashboard
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Created - Created container: rhods-dashboard
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 592ms (592ms including waiting). Image size: 812543307 bytes.
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 11:34:22   11:31:08 INFO Pod/rhods-dashboard-775867476c-6thcb: AddedInterface - Add eth0 [10.134.0.128/23] from ovn-kubernetes
2026/09/07 11:34:22   11:31:07 INFO Pod/rhods-dashboard-775867476c-6thcb: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-775867476c-6thcb to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:31:07 INFO Pod/eval-hub-ui-869b89d9b4-fnhth: Killing - Stopping container eval-hub-ui
2026/09/07 11:34:22   11:31:07 INFO ReplicaSet/rhods-dashboard-775867476c: SuccessfulCreate - Created pod: rhods-dashboard-775867476c-6thcb
2026/09/07 11:34:22   11:31:00 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-jmr46: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:53 INFO Pod/kserve-module-controller-manager-66ff7c977-5qrvv: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:51 INFO Lease/b7e9931f.trustyai.opendatahub.io: LeaderElection - trustyai-service-operator-controller-manager-d6bf47757-jmr46_6e6c16d4-efac-45d8-ba01-9deb23e0bacc became leader
2026/09/07 11:34:22   11:30:50 INFO Pod/kserve-controller-manager-5fbbd6dd88-4jglj: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:50 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-sl9nr: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:50 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-6vpxx: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:50 INFO Pod/odh-model-controller-57457c4c89-kt7sg: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:50 INFO Pod/model-serving-api-7df8c9655f-rt45n: Killing - Stopping container server
2026/09/07 11:34:22   11:30:49 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:30:49 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 11:34:22   11:30:49 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-n2qlk
2026/09/07 11:34:22   11:30:49 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-lwzt2
2026/09/07 11:34:22   11:30:49 INFO Pod/kserve-localmodelnode-agent-lwzt2: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:49 INFO Pod/kserve-localmodelnode-agent-n2qlk: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:35 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-jmr46: AddedInterface - Add eth0 [10.134.0.127/23] from ovn-kubernetes
2026/09/07 11:34:22   11:30:35 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-jmr46: Started - Started container manager
2026/09/07 11:34:22   11:30:35 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-jmr46: Created - Created container: manager
2026/09/07 11:34:22   11:30:35 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-jmr46: Pulled - Container image "quay.io/rhoai/odh-trustyai-service-operator-rhel9@sha256:e41a5793f10820747e807d9165c73298387d7feefc81f4b0d9b70e911d6dbe40" already present on machine
2026/09/07 11:34:22   11:30:34 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-jmr46: Scheduled - Successfully assigned redhat-ods-applications/trustyai-service-operator-controller-manager-d6bf47757-jmr46 to ip-10-0-142-168.ec2.internal
2026/09/07 11:34:22   11:30:34 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-j9c45: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:34 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-nmfrj: Killing - Stopping container manager
2026/09/07 11:34:22   11:30:34 INFO ReplicaSet/trustyai-service-operator-controller-manager-d6bf47757: SuccessfulCreate - Created pod: trustyai-service-operator-controller-manager-d6bf47757-jmr46
2026/09/07 11:34:22   11:30:34 INFO Deployment/trustyai-service-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trustyai-service-operator-controller-manager-d6bf47757 from 0 to 1
2026/09/07 11:34:22   11:29:41 WARN Pod/rhods-dashboard-85f74c5bcd-vhxz2: Unhealthy - Readiness probe failed: Get "https://10.134.0.119:8444/healthz": dial tcp 10.134.0.119:8444: connect: connection refused
2026/09/07 11:34:22   11:29:36 WARN Pod/rhods-dashboard-85f74c5bcd-vhxz2: Unhealthy - Readiness probe failed: Get "https://10.134.0.119:8943/healthcheck": dial tcp 10.134.0.119:8943: connect: connection refused
2026/09/07 11:34:22   11:29:36 WARN Pod/rhods-dashboard-85f74c5bcd-vhxz2: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 11:34:22 === RESOURCE QUOTAS ===
2026/09/07 11:34:22 No resource quotas found in monitored namespaces
2026/09/07 11:34:22 Classification for components: infrastructure/storage (code=1006, confidence=medium)
2026/09/07 11:34:22   Evidence: event Pod/kserve-localmodelnode-agent-zfds9: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"storage","error_code":1006,"evidence":["event Pod/kserve-localmodelnode-agent-zfds9: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found"],"confidence":"medium"}
2026/09/07 11:34:22 Diagnostics complete for failed test: components
    controller_test.go:490: Stopping: components test failed.
2026/09/07 11:34:22 Circuit breaker: failure classified as infrastructure/storage (code=1006)
2026/09/07 11:34:22 Circuit breaker: infrastructure/unknown failure recorded (2/3)
    controller_test.go:762: Skipping tests for services/monitoring
Deletion Policy: Never. Skipping deletion tests.
2026/09/07 11:34:23 Circuit breaker: no trips during this run (threshold=3)
    controller_test.go:762: Skipping tests for services/monitoring]]></system-out>
	</testsuite>
</testsuites>
