<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="329" failures="1" skipped="93">
	<testsuite name="" tests="329" failures="1" errors="0" id="0" hostname="e2e-cli-nmanos-ephc-rhoai-smoke-x7l8f-test-platform-pod" skipped="93" time="7471.260" timestamp="2026-09-06T18:16:17Z">
		<testcase name="TestOdhOperator" classname="" time="1078.820">
			<system-out><![CDATA[2026/09/06 17:58:18 ARTIFACT_DIR not set, skipping periodic metrics collection
2026-09-06T17:58:18Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests" classname="" time="0.190">
			<system-out><![CDATA[2026-09-06T17:58:18Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Datascience_Cluster_CRD" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataScienceCluster_Initialization_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_FeatureTracker_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Ray_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrustyAI_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Kueue_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataSciencePipelines_CRD" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Auth_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Platform_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests" classname="" time="0.480">
			<system-out><![CDATA[2026-09-06T17:58:19Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_HardwareProfile_resource" classname="" time="10.130"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_owned_namespaces_exist" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_default_NetworkPolicy_exist" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_denies_invalid_PEM_on_update" classname="" time="0.100"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_valid_PEM_on_update" classname="" time="0.140"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_invalid_PEM_when_Removed" classname="" time="0.110"></testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests" classname="" time="11.150">
			<system-out><![CDATA[2026-09-06T17:58:29Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Ensuring clean slate: setting all components to Removed
    controller_test.go:762: Patched Subscription to remove env vars: [RHAI_VERSION CI]]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_in-tree_gates_block_and_unblock_provisioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_admin_ack_gates_block_and_unblock_provisioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_partial_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_runlevel_gating_and_convergence" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_PlatformReady_condition_on_component_CRs" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_component_stability_across_enable/disable_cycles_(RHOAIENG-73142)" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_DAG_cleanup" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/monitoring" classname="" time="0.150">
			<skipped message="Skipped"><![CDATA[2026-09-06T17:58:40Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components" classname="" time="1054.910"></testcase>
		<testcase name="TestOdhOperator/components/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway" classname="" time="70.750"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard" classname="" time="182.450">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines" classname="" time="86.550"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator" classname="" time="66.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve" classname="" time="454.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator" classname="" time="51.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx" classname="" time="70.690"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray" classname="" time="91.350">
			<system-out><![CDATA[2026-09-06T17:59:47Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator" classname="" time="60.690">
			<system-out><![CDATA[2026-09-06T17:59:32Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer" classname="" time="40.680">
			<system-out><![CDATA[2026-09-06T17:59:32Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches" classname="" time="51.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_enabled" classname="" time="60.340">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_enabled" classname="" time="40.670">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_enabled" classname="" time="101.450"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_enabled" classname="" time="40.630">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_enabled" classname="" time="60.400">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_component_enabled" classname="" time="40.720">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade_from_in-tree:_selector_migration" classname="" time="0.030">
			<system-out><![CDATA[2026-09-06T17:58:41Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_enabled" classname="" time="20.510"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_component_enabled" classname="" time="0.550"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_operator_deployed" classname="" time="25.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_conditions" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources" classname="" time="20.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources/deployment_data-science-pipelines-operator-controller-manager" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_created" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_ready" classname="" time="30.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_operator_deployment" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_workbenches_namespace_configuration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_ImageStreams_available" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_MLflow_integration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_WorkbenchesV2_default_Removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery" classname="" time="0.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_argoWorkflowsControllers_options" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery" classname="" time="35.490"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="15.200"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_disabled" classname="" time="10.140">
			<system-out><![CDATA[    helper_test.go:156: No ConfigMap resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-config" classname="" time="5.060">
			<system-out><![CDATA[    helper_test.go:156: No Service resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_operands_have_OwnerReferences" classname="" time="0.050">
			<system-out><![CDATA[    helper_test.go:156: No Deployment resources found for component Workbenches, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_disabled" classname="" time="10.260"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_update_operand_resources" classname="" time="20.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_workflow-controller-configmap" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_enabled" classname="" time="30.280"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_WorkbenchesV2_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_enabled" classname="" time="50.320"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_feast-operator_deployed_by_module" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade:_existing_operands_preserved" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_disabled_cleanup" classname="" time="10.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="5.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_data-science-pipelines-operator-service" classname="" time="5.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_disabled" classname="" time="10.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_with_mixed_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_disabled" classname="" time="10.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_dynamically_watches_operands" classname="" time="10.200"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.160"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_data-science-pipelines-operator-controller-manager" classname="" time="15.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_enabled" classname="" time="30.510"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_CRDs_reinstated" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_VAP_blocks_dashboard_HardwareProfile_and_AcceleratorProfile_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery" classname="" time="111.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="100.670"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_agent-ops-params" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_module_handler_projects_DSC_config_to_Module_CR" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_disabled" classname="" time="10.260"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_automl-params" classname="" time="95.080">
			<failure message="Failed"></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources/deployment_kuberay-operator" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_spec" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_operator_bootstrap_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_SparkPi_workload_execution" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_ScheduledSparkApplication_workload_execution" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_operator_Deployment_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_disabled" classname="" time="10.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_no_Kserve_FeatureTrackers" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_VAP_created_when_kserve_is_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources" classname="" time="100.200"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery" classname="" time="30.330"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_ray-config-bf2bfgg6b4" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-localmodel-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="10.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-operator" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-webhook-service" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kuberay-operator" classname="" time="15.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_llmisvc-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_disabled" classname="" time="10.230"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_model-serving-api" classname="" time="20.040">
			<system-out><![CDATA[    test_context_test.go:1326: 
        Timed out after 90.004s.
        Resource ConfigMap redhat-ods-applications/automl-params was not properly recreated with new identity
        The function passed to Eventually failed at /workspace/tests/e2e/test_context_test.go:1307 with:
        Expected ConfigMap redhat-ods-applications/automl-params to be recreated
        Expected
            <*unstructured.Unstructured | 0x0>: nil
        not to be nil
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_odh-model-controller" classname="" time="20.040">
			<system-out><![CDATA[2026/09/06 18:01:42 === CLUSTER STATE ===
2026/09/06 18:01:42 Node: ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:01:42   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:01:42   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:01:42   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:01:42   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:01:42 Node: ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:01:42   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:01:42   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:01:42   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:01:42   Resources: 7500m CPU, 30892144Ki memory allocatable (of 8000m CPU, 32043120Ki memory capacity)
2026/09/06 18:01:42 Node: ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:01:42   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:01:42   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:01:42   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:01:42   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:01:42 === DEPLOYMENTS ===
2026/09/06 18:01:42 Namespace: redhat-ods-applications
2026/09/06 18:01:42   Deployment agent-ops-ui: 1/1 (READY)
2026/09/06 18:01:42   Deployment dashboard-operator: 2/2 (READY)
2026/09/06 18:01:42   Deployment dashboard-redirect: 2/2 (READY)
2026/09/06 18:01:42   Deployment gen-ai-ui: 1/1 (READY)
2026/09/06 18:01:42   Deployment kserve-controller-manager: 2/2 (READY)
2026/09/06 18:01:42   Deployment kserve-localmodel-controller-manager: 2/2 (READY)
2026/09/06 18:01:42   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/06 18:01:42   Deployment llmisvc-controller-manager: 2/2 (READY)
2026/09/06 18:01:42   Deployment maas-ui: 1/1 (READY)
2026/09/06 18:01:42   Deployment model-serving-api: 2/2 (READY)
2026/09/06 18:01:42   Deployment notebooks-ui: 1/1 (READY)
2026/09/06 18:01:42   Deployment odh-model-controller: 1/1 (READY)
2026/09/06 18:01:42   Deployment rhods-dashboard: 2/2 (READY)
2026/09/06 18:01:42 Namespace: kube-system
2026/09/06 18:01:42 === PODS ===
2026/09/06 18:01:42 Failed to collect pod data: redhat-ods-applications/rhods-dashboard-7b658c686f-29gd9: container kube-rbac-proxy not ready
2026/09/06 18:01:42 Continuing with partial pod data to collect logs from problematic containers
2026/09/06 18:01:42 Namespace: redhat-ods-applications
2026/09/06 18:01:42   Pod rhods-dashboard-7b658c686f-29gd9: Running
2026/09/06 18:01:42     Container kube-rbac-proxy: ready=false
2026/09/06 18:01:42 Namespace kube-system: no problematic pods found
2026/09/06 18:01:42 === OPERATOR STATUS ===
2026/09/06 18:01:42 Operator deployment: 3/3 ready
2026/09/06 18:01:42 === DSCI STATUS ===
2026/09/06 18:01:42 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/06 18:01:42 === DSC STATUS ===
2026/09/06 18:01:42 DSC default-dsc:
2026/09/06 18:01:42   Ready: True - 
2026/09/06 18:01:42   ProvisioningSucceeded: True - 
2026/09/06 18:01:42   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/06 18:01:42   ModulesReady: True - 
2026/09/06 18:01:42   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:42   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:01:42   DashboardReady: True - 
2026/09/06 18:01:42   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   KserveLLMInferenceServiceDependencies: True - 
2026/09/06 18:01:42   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/06 18:01:42   KserveReady: True - 
2026/09/06 18:01:42   KueueReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:42   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:01:42   OGXReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   RayReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:42   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   TrainerReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:42   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:42   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/06 18:01:42 === RECENT EVENTS ===
2026/09/06 18:01:42   18:01:40 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8444/healthz": dial tcp 10.132.0.115:8444: connect: connection refused
2026/09/06 18:01:42   18:01:25 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7b658c686f from 1 to 0
2026/09/06 18:01:42   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:42   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Started - Started container server
2026/09/06 18:01:42   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Created - Created container: server
2026/09/06 18:01:42   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.56s (1.56s including waiting). Image size: 172459788 bytes.
2026/09/06 18:01:42   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container core-bff
2026/09/06 18:01:42   18:01:25 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:42   18:01:25 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8943/healthcheck": dial tcp 10.132.0.115:8943: connect: connection refused
2026/09/06 18:01:42   18:01:25 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulDelete - Deleted pod: rhods-dashboard-7b658c686f-29gd9
2026/09/06 18:01:42   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:42   18:01:23 INFO Pod/model-serving-api-7df8c9655f-4j78j: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/06 18:01:42   18:01:23 INFO Pod/model-serving-api-7df8c9655f-4j78j: AddedInterface - Add eth0 [10.132.0.127/23] from ovn-kubernetes
2026/09/06 18:01:42   18:01:22 INFO Pod/model-serving-api-7df8c9655f-4j78j: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-4j78j to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   18:01:22 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/06 18:01:42   18:01:22 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-4j78j
2026/09/06 18:01:42   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:42   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe failed: Get "https://10.133.0.110:8943/healthcheck": dial tcp 10.133.0.110:8943: connect: connection refused
2026/09/06 18:01:42   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe failed: Get "https://10.133.0.110:8444/healthz": dial tcp 10.133.0.110:8444: connect: connection refused
2026/09/06 18:01:42   18:01:08 INFO Pod/kuberay-operator-7d87f54798-s45kr: Killing - Stopping container kuberay-operator
2026/09/06 18:01:42   18:01:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Created - Created container: manager
2026/09/06 18:01:42   18:01:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Started - Started container manager
2026/09/06 18:01:42   18:01:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 2.257s (2.257s including waiting). Image size: 235279660 bytes.
2026/09/06 18:01:42   18:01:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:01:42   18:01:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: AddedInterface - Add eth0 [10.132.0.126/23] from ovn-kubernetes
2026/09/06 18:01:42   18:01:02 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-h588p to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   18:01:02 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-h588p
2026/09/06 18:01:42   18:01:02 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-s45kr to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   18:00:53 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-s45kr
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Started - Started container kuberay-operator
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Killing - Stopping container kuberay-operator
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: AddedInterface - Add eth0 [10.132.0.125/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Killing - Stopping container kuberay-operator
2026/09/06 18:01:42   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Created - Created container: kuberay-operator
2026/09/06 18:01:42   18:00:53 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Started - Started container kube-rbac-proxy
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Started - Started container rhods-dashboard
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Created - Created container: core-bff
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Started - Started container core-bff
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Created - Created container: rhods-dashboard
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 350ms (350ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:42   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 387ms (387ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:42   18:00:47 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-86c5fb98b4-wn46m to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   18:00:47 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulDelete - Deleted pod: rhods-dashboard-797b4b4bd7-9c6vt
2026/09/06 18:01:42   18:00:47 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-797b4b4bd7 from 1 to 0
2026/09/06 18:01:42   18:00:47 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-86c5fb98b4 from 1 to 2
2026/09/06 18:01:42   18:00:47 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: AddedInterface - Add eth0 [10.133.0.122/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:47 INFO ReplicaSet/rhods-dashboard-86c5fb98b4: SuccessfulCreate - Created pod: rhods-dashboard-86c5fb98b4-wn46m
2026/09/06 18:01:42   18:00:47 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:42   18:00:47 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Killing - Stopping container core-bff
2026/09/06 18:01:42   18:00:47 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:42   18:00:47 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:42   18:00:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 2.063s (2.063s including waiting). Image size: 244249302 bytes.
2026/09/06 18:01:42   18:00:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Created - Created container: manager
2026/09/06 18:01:42   18:00:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Started - Started container manager
2026/09/06 18:01:42   18:00:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: AddedInterface - Add eth0 [10.133.0.121/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:01:42   18:00:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-944q2 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   18:00:42 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-944q2
2026/09/06 18:01:42   18:00:42 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 1 to 2
2026/09/06 18:01:42   18:00:37 WARN Pod/rhods-dashboard-7b658c686f-wn6p5: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:42   18:00:37 WARN Pod/rhods-dashboard-7b658c686f-wn6p5: Unhealthy - Readiness probe failed: Get "https://10.133.0.118:8943/healthcheck": dial tcp 10.133.0.118:8943: connect: connection refused
2026/09/06 18:01:42   18:00:34 WARN Pod/rhods-dashboard-7b658c686f-wn6p5: Unhealthy - Readiness probe failed: Get "https://10.133.0.118:8444/healthz": dial tcp 10.133.0.118:8444: connect: connection refused
2026/09/06 18:01:42   18:00:25 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Started - Started container manager
2026/09/06 18:01:42   18:00:25 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Created - Created container: manager
2026/09/06 18:01:42   18:00:25 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 2.402s (2.402s including waiting). Image size: 247667445 bytes.
2026/09/06 18:01:42   18:00:23 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Killing - Stopping container manager
2026/09/06 18:01:42   18:00:23 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:01:42   18:00:23 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: AddedInterface - Add eth0 [10.134.0.83/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:22 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-h65m7 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   18:00:22 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Killing - Stopping container controller
2026/09/06 18:01:42   18:00:22 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Killing - Stopping container webhook
2026/09/06 18:01:42   18:00:22 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 1 to 2
2026/09/06 18:01:42   18:00:22 WARN ReplicaSet/spark-operator-webhook-5d466b7d66: FailedCreate - Error creating: pods "spark-operator-webhook-5d466b7d66-" is forbidden: error looking up service account redhat-ods-applications/spark-operator-webhook: serviceaccount "spark-operator-webhook" not found
2026/09/06 18:01:42   18:00:22 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/06 18:01:42   18:00:22 WARN Pod/rhods-dashboard-797b4b4bd7-4k4st: Unhealthy - Readiness probe failed: Get "https://10.132.0.108:8444/healthz": dial tcp 10.132.0.108:8444: connect: connection refused
2026/09/06 18:01:42   18:00:22 INFO Lease/spark-operator-controller-lock: LeaderElection - spark-operator-controller-c594b967f-mx9ls_69e073cb-90ee-454b-9975-f067656c5991 became leader
2026/09/06 18:01:42   18:00:22 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-h65m7
2026/09/06 18:01:42   18:00:21 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-5d466b7d66-zz4s5_44409d15-d15a-495f-b780-470d2e1a509d became leader
2026/09/06 18:01:42   18:00:20 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Started - Started container kuberay-operator
2026/09/06 18:01:42   18:00:20 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Created - Created container: kuberay-operator
2026/09/06 18:01:42   18:00:19 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.498s (1.498s including waiting). Image size: 189730715 bytes.
2026/09/06 18:01:42   18:00:19 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7d87f54798-l5rl9_f041aff2-fca6-4e53-aa64-1cda3cabfc48 became leader
2026/09/06 18:01:42   18:00:18 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/06 18:01:42   18:00:18 INFO Pod/kuberay-operator-7d87f54798-wxzdq: AddedInterface - Add eth0 [10.134.0.82/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:17 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-wxzdq to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   18:00:17 WARN Pod/rhods-dashboard-797b4b4bd7-4k4st: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:42   18:00:17 WARN Pod/rhods-dashboard-797b4b4bd7-4k4st: Unhealthy - Readiness probe failed: Get "https://10.132.0.108:8943/healthcheck": dial tcp 10.132.0.108:8943: connect: connection refused
2026/09/06 18:01:42   18:00:17 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-wxzdq
2026/09/06 18:01:42   18:00:17 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 1 to 2
2026/09/06 18:01:42   18:00:09 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Started - Started container core-bff
2026/09/06 18:01:42   18:00:09 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 542ms (542ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:42   18:00:09 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Created - Created container: core-bff
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: AddedInterface - Add eth0 [10.132.0.123/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Started - Started container rhods-dashboard
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Started - Started container kube-rbac-proxy
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 377ms (377ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:42   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Created - Created container: rhods-dashboard
2026/09/06 18:01:42   18:00:07 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-86c5fb98b4-lvzfm to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   18:00:07 INFO ReplicaSet/rhods-dashboard-86c5fb98b4: SuccessfulCreate - Created pod: rhods-dashboard-86c5fb98b4-lvzfm
2026/09/06 18:01:42   18:00:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7b658c686f from 2 to 1
2026/09/06 18:01:42   18:00:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-86c5fb98b4 from 0 to 1
2026/09/06 18:01:42   18:00:07 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulDelete - Deleted pod: rhods-dashboard-7b658c686f-wn6p5
2026/09/06 18:01:42   18:00:07 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:42   18:00:07 INFO Pod/automl-ui-74d9b97d49-x68qc: Killing - Stopping container automl-ui
2026/09/06 18:01:42   18:00:07 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Killing - Stopping container core-bff
2026/09/06 18:01:42   18:00:07 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:42   18:00:07 INFO Pod/autorag-ui-58f548cbd-z4g4s: Killing - Stopping container autorag-ui
2026/09/06 18:01:42   18:00:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   18:00:04 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-webhook-5d466b7d66-zz4s5 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-controller-c594b967f-mx9ls to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   18:00:04 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: AddedInterface - Add eth0 [10.133.0.120/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:04 INFO Deployment/spark-operator-controller: ScalingReplicaSet - Scaled up replica set spark-operator-controller-c594b967f from 0 to 1
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Started - Started container webhook
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Created - Created container: webhook
2026/09/06 18:01:42   18:00:04 INFO ReplicaSet/spark-operator-webhook-5d466b7d66: SuccessfulCreate - Created pod: spark-operator-webhook-5d466b7d66-zz4s5
2026/09/06 18:01:42   18:00:04 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Killing - Stopping container manager
2026/09/06 18:01:42   18:00:04 INFO ReplicaSet/spark-operator-controller-c594b967f: SuccessfulCreate - Created pod: spark-operator-controller-c594b967f-mx9ls
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: AddedInterface - Add eth0 [10.133.0.119/23] from ovn-kubernetes
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Started - Started container controller
2026/09/06 18:01:42   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Created - Created container: controller
2026/09/06 18:01:42   18:00:03 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   18:00:03 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   18:00:03 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   18:00:03 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   18:00:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Killing - Stopping container manager
2026/09/06 18:01:42   18:00:01 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-drj8q_49a3cad5-9238-4bb4-853f-095d4e050c85 became leader
2026/09/06 18:01:42   18:00:00 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Started - Started container kuberay-operator
2026/09/06 18:01:42   18:00:00 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Created - Created container: kuberay-operator
2026/09/06 18:01:42   18:00:00 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.908s (1.908s including waiting). Image size: 189730715 bytes.
2026/09/06 18:01:42   18:00:00 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-rtg5q_5ed5f5a4-56d7-43a3-b972-c60b4ae9b685 became leader
2026/09/06 18:01:42   17:59:59 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-jdmf2_4cdcf91a-008a-4a24-b6d1-c3b3814a8d19 became leader
2026/09/06 18:01:42   17:59:58 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/06 18:01:42   17:59:58 INFO Pod/kuberay-operator-7d87f54798-l5rl9: AddedInterface - Add eth0 [10.132.0.122/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:57 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:57 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:57 WARN Pod/kuberay-operator-7d87f54798-l5rl9: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/06 18:01:42   17:59:57 INFO Lease/spark-operator-module-controller-manager: LeaderElection - spark-operator-module-controller-manager-59549b5854-69mlg_2ef4d6ec-bc98-48df-997a-6715d3827d7a became leader
2026/09/06 18:01:42   17:59:57 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 274ms (274ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:42   17:59:57 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:57 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Created - Created container: core-bff
2026/09/06 18:01:42   17:59:57 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Started - Started container core-bff
2026/09/06 18:01:42   17:59:56 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-l5rl9 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:42   17:59:56 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-l5rl9
2026/09/06 18:01:42   17:59:56 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Created - Created container: rhods-dashboard
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 289ms (289ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Started - Started container kube-rbac-proxy
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: AddedInterface - Add eth0 [10.133.0.118/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Started - Started container rhods-dashboard
2026/09/06 18:01:42   17:59:55 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7b658c686f-wn6p5 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:55 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:42   17:59:55 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Killing - Stopping container core-bff
2026/09/06 18:01:42   17:59:55 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulCreate - Created pod: rhods-dashboard-7b658c686f-wn6p5
2026/09/06 18:01:42   17:59:55 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7b658c686f from 1 to 2
2026/09/06 18:01:42   17:59:55 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-797b4b4bd7 from 2 to 1
2026/09/06 18:01:42   17:59:55 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulDelete - Deleted pod: rhods-dashboard-797b4b4bd7-4k4st
2026/09/06 18:01:42   17:59:55 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-bpwt8_f7565ff5-f964-4439-8661-a4823fd2bbaf became leader
2026/09/06 18:01:42   17:59:55 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:42   17:59:51 INFO Pod/kserve-localmodelnode-agent-nh54p: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 436ms (436ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:01:42   17:59:51 INFO Pod/kserve-localmodelnode-agent-nh54p: Created - Created container: manager
2026/09/06 18:01:42   17:59:51 INFO Pod/kserve-localmodelnode-agent-nh54p: Started - Started container manager
2026/09/06 18:01:42   17:59:51 INFO Pod/kserve-localmodelnode-agent-mmmpl: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 447ms (447ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:01:42   17:59:51 INFO Pod/kserve-localmodelnode-agent-mmmpl: Created - Created container: manager
2026/09/06 18:01:42   17:59:51 INFO Pod/kserve-localmodelnode-agent-mmmpl: Started - Started container manager
2026/09/06 18:01:42   17:59:50 INFO Pod/kserve-localmodelnode-agent-mmmpl: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-mmmpl to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:59:50 INFO Pod/kserve-localmodelnode-agent-nh54p: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-nh54p to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:50 INFO Pod/kserve-localmodelnode-agent-mmmpl: AddedInterface - Add eth0 [10.134.0.80/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:50 INFO Pod/kserve-localmodelnode-agent-mmmpl: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:01:42   17:59:50 INFO Pod/kserve-localmodelnode-agent-nh54p: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:01:42   17:59:50 INFO Pod/kserve-localmodelnode-agent-nh54p: AddedInterface - Add eth0 [10.133.0.117/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:44 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 648ms (648ms including waiting). Image size: 244249302 bytes.
2026/09/06 18:01:42   17:59:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Started - Started container manager
2026/09/06 18:01:42   17:59:43 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:43 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Created - Created container: manager
2026/09/06 18:01:42   17:59:43 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 540ms (540ms including waiting). Image size: 239132625 bytes.
2026/09/06 18:01:42   17:59:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/06 18:01:42   17:59:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Created - Created container: manager
2026/09/06 18:01:42   17:59:43 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Started - Started container manager
2026/09/06 18:01:42   17:59:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Started - Started container manager
2026/09/06 18:01:42   17:59:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Created - Created container: manager
2026/09/06 18:01:42   17:59:42 WARN Pod/kserve-localmodelnode-agent-mmmpl: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/06 18:01:42   17:59:42 WARN Pod/kserve-localmodelnode-agent-nh54p: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/06 18:01:42   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89xhn65 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:59:42 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Started - Started container manager
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: AddedInterface - Add eth0 [10.132.0.120/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: AddedInterface - Add eth0 [10.133.0.114/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:01:42   17:59:42 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2":
 Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:01:42   17:59:42 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "host
mount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:01:42   17:59:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-nh54p
2026/09/06 18:01:42   17:59:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-mmmpl
2026/09/06 18:01:42   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Created - Created container: manager
2026/09/06 18:01:42   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 370ms (370ms including waiting). Image size: 235279660 bytes.
2026/09/06 18:01:42   17:59:42 INFO Pod/odh-model-controller-57457c4c89-rtg5q: AddedInterface - Add eth0 [10.132.0.121/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:42 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 390ms (390ms including waiting). Image size: 247667445 bytes.
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Created - Created container: manager
2026/09/06 18:01:42   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:42 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89xhn65
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Started - Started container manager
2026/09/06 18:01:42   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Started - Started container server
2026/09/06 18:01:42   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Created - Created container: server
2026/09/06 18:01:42   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/06 18:01:42   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:01:42   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:42 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: AddedInterface - Add eth0 [10.134.0.79/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:42 INFO Pod/maas-controller-56c898799-m5mvz: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:01:42   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:42 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/06 18:01:42   17:59:41 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:41 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-rtg5q to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:41 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-k4nbq to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:41 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-jdmf2 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:41 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-5g8dz to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:41 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-drj8q to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:41 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/06 18:01:42   17:59:41 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/06 18:01:42   17:59:41 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt_ac428613-9176-4ff3-ace8-0d78242b4abc became leader
2026/09/06 18:01:42   17:59:41 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/06 18:01:42   17:59:41 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-jdmf2
2026/09/06 18:01:42   17:59:41 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/06 18:01:42   17:59:41 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-k4nbq
2026/09/06 18:01:42   17:59:41 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/06 18:01:42   17:59:41 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-rtg5q
2026/09/06 18:01:42   17:59:41 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-drj8q
2026/09/06 18:01:42   17:59:41 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-5g8dz
2026/09/06 18:01:42   17:59:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Started - Started container manager
2026/09/06 18:01:42   17:59:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Created - Created container: manager
2026/09/06 18:01:42   17:59:40 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Started - Started container manager
2026/09/06 18:01:42   17:59:39 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" already present on machine
2026/09/06 18:01:42   17:59:39 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Created - Created container: manager
2026/09/06 18:01:42   17:59:39 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Pulled - Successfully pulled image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637" in 2.753s (2.753s including waiting). Image size: 191954597 bytes.
2026/09/06 18:01:42   17:59:38 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:38 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-bpwt8 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:38 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:38 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: AddedInterface - Add eth0 [10.133.0.113/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:38 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-bpwt8
2026/09/06 18:01:42   17:59:38 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-56c898799-m5mvz_82a9a590-d174-4f56-9ec0-3a18dd218dcc became leader
2026/09/06 18:01:42   17:59:38 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/06 18:01:42   17:59:37 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Pulling - Pulling image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637"
2026/09/06 18:01:42   17:59:37 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:37 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: AddedInterface - Add eth0 [10.133.0.112/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:36 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-module-controller-manager-59549b5854-69mlg to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:36 INFO Deployment/spark-operator-module-controller-manager: ScalingReplicaSet - Scaled up replica set spark-operator-module-controller-manager-59549b5854 from 0 to 1
2026/09/06 18:01:42   17:59:36 INFO ReplicaSet/spark-operator-module-controller-manager-59549b5854: SuccessfulCreate - Created pod: spark-operator-module-controller-manager-59549b5854-69mlg
2026/09/06 18:01:42   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-lgmzn to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Pulled - Container image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" already present on machine
2026/09/06 18:01:42   17:59:33 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/06 18:01:42   17:59:33 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-lgmzn
2026/09/06 18:01:42   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: AddedInterface - Add eth0 [10.133.0.111/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Created - Created container: manager
2026/09/06 18:01:42   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Started - Started container manager
2026/09/06 18:01:42   17:59:27 INFO Pod/workbenches-operator-c674d9c9-485g4: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:25 WARN Pod/notebook-controller-deployment-75c8689979-2s8r8: Unhealthy - Readiness probe failed: Get "http://10.133.0.109:8081/readyz": dial tcp 10.133.0.109:8081: connect: connection refused
2026/09/06 18:01:42   17:59:25 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:25 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:24 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:22 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Created - Created container: manager
2026/09/06 18:01:42   17:59:22 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2" in 2.825s (2.825s including waiting). Image size: 193888070 bytes.
2026/09/06 18:01:42   17:59:22 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Started - Started container manager
2026/09/06 18:01:42   17:59:22 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Started - Started container manager
2026/09/06 18:01:42   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: Started - Started container manager
2026/09/06 18:01:42   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: Created - Created container: manager
2026/09/06 18:01:42   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" already present on machine
2026/09/06 18:01:42   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: AddedInterface - Add eth0 [10.132.0.119/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:22 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Created - Created container: manager
2026/09/06 18:01:42   17:59:22 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7" in 3.513s (3.513s including waiting). Image size: 184745870 bytes.
2026/09/06 18:01:42   17:59:21 INFO Pod/dashboard-operator-669d6564bd-zmng8: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-zmng8 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:21 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:21 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 1 to 2
2026/09/06 18:01:42   17:59:21 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:21 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-zmng8
2026/09/06 18:01:42   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Created - Created container: manager
2026/09/06 18:01:42   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd" in 662ms (662ms including waiting). Image size: 197531122 bytes.
2026/09/06 18:01:42   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd"
2026/09/06 18:01:42   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Started - Started container manager
2026/09/06 18:01:42   17:59:19 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: AddedInterface - Add eth0 [10.132.0.118/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: AddedInterface - Add eth0 [10.132.0.116/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:19 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2"
2026/09/06 18:01:42   17:59:18 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:18 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Pulling - Pulling image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7"
2026/09/06 18:01:42   17:59:18 WARN Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "ogx-k8s-operator-webhook-cert" not found
2026/09/06 18:01:42   17:59:18 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: AddedInterface - Add eth0 [10.132.0.117/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:18 INFO ReplicaSet/ogx-k8s-operator-controller-manager-59777cdbbb: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt
2026/09/06 18:01:42   17:59:17 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-5fc6bb8d67-x498g to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:17 INFO Pod/maas-controller-56c898799-m5mvz: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-56c898799-m5mvz to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:17 WARN Pod/maas-controller-56c898799-m5mvz: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "maas-controller-webhook-cert" not found
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Created - Created container: core-bff
2026/09/06 18:01:42   17:59:17 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-5fc6bb8d67 from 0 to 1
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Started - Started container core-bff
2026/09/06 18:01:42   17:59:17 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-56c898799 from 0 to 1
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 334ms (334ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Started - Started container kube-rbac-proxy
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Started - Started container rhods-dashboard
2026/09/06 18:01:42   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Created - Created container: rhods-dashboard
2026/09/06 18:01:42   17:59:17 INFO ReplicaSet/feast-operator-controller-manager-5fc6bb8d67: SuccessfulCreate - Created pod: feast-operator-controller-manager-5fc6bb8d67-x498g
2026/09/06 18:01:42   17:59:17 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/06 18:01:42   17:59:17 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-59777cdbbb from 0 to 1
2026/09/06 18:01:42   17:59:17 INFO ReplicaSet/maas-controller-56c898799: SuccessfulCreate - Created pod: maas-controller-56c898799-m5mvz
2026/09/06 18:01:42   17:59:16 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Started - Started container agent-ops-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/notebooks-ui-6577cc467-grs8s: Started - Started container notebooks-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/maas-ui-74675d48f9-wjrcd: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe" in 3.918s (3.918s including waiting). Image size: 189581000 bytes.
2026/09/06 18:01:42   17:59:16 INFO Pod/notebooks-ui-6577cc467-grs8s: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003" in 3.772s (3.772s including waiting). Image size: 190593254 bytes.
2026/09/06 18:01:42   17:59:16 INFO Pod/notebooks-ui-6577cc467-grs8s: Created - Created container: notebooks-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/rhods-dashboard-7b658c686f-29gd9: AddedInterface - Add eth0 [10.132.0.115/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:16 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987" in 4.949s (4.949s including waiting). Image size: 238860397 bytes.
2026/09/06 18:01:42   17:59:16 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:42   17:59:16 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Started - Started container gen-ai-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Created - Created container: gen-ai-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/automl-ui-74d9b97d49-x68qc: Started - Started container automl-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/maas-ui-74675d48f9-wjrcd: Started - Started container maas-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3" in 3.579s (3.579s including waiting). Image size: 184646446 bytes.
2026/09/06 18:01:42   17:59:16 INFO Pod/autorag-ui-58f548cbd-z4g4s: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2" in 5.263s (5.264s including waiting). Image size: 223947455 bytes.
2026/09/06 18:01:42   17:59:16 INFO Pod/autorag-ui-58f548cbd-z4g4s: Created - Created container: autorag-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/autorag-ui-58f548cbd-z4g4s: Started - Started container autorag-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 345ms (345ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:42   17:59:16 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Created - Created container: agent-ops-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/maas-ui-74675d48f9-wjrcd: Created - Created container: maas-ui
2026/09/06 18:01:42   17:59:16 INFO Pod/automl-ui-74d9b97d49-x68qc: Created - Created container: automl-ui
2026/09/06 18:01:42   17:59:15 INFO Pod/automl-ui-74d9b97d49-x68qc: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af" in 4.15s (4.15s including waiting). Image size: 222806700 bytes.
2026/09/06 18:01:42   17:59:15 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Killing - Stopping container manager
2026/09/06 18:01:42   17:59:13 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-q5z75_08d79772-5a7f-4b27-a7e8-ca0d73ebaaba became leader
2026/09/06 18:01:42   17:59:12 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7b658c686f-29gd9 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:12 INFO Pod/maas-ui-74675d48f9-wjrcd: AddedInterface - Add eth0 [10.132.0.112/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:12 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: AddedInterface - Add eth0 [10.132.0.113/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:12 INFO Pod/notebooks-ui-6577cc467-grs8s: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003"
2026/09/06 18:01:42   17:59:12 INFO Pod/maas-ui-74675d48f9-wjrcd: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe"
2026/09/06 18:01:42   17:59:12 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulCreate - Created pod: rhods-dashboard-7b658c686f-29gd9
2026/09/06 18:01:42   17:59:12 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7b658c686f from 0 to 1
2026/09/06 18:01:42   17:59:12 INFO Pod/notebooks-ui-6577cc467-grs8s: AddedInterface - Add eth0 [10.132.0.114/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:12 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3"
2026/09/06 18:01:42   17:59:11 INFO Pod/notebooks-ui-6577cc467-grs8s: Scheduled - Successfully assigned redhat-ods-applications/notebooks-ui-6577cc467-grs8s to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:11 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-7f9fbf68-lq7cv to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:11 INFO Pod/maas-ui-74675d48f9-wjrcd: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-74675d48f9-wjrcd to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:11 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-74675d48f9 from 0 to 1
2026/09/06 18:01:42   17:59:11 INFO ReplicaSet/agent-ops-ui-7f9fbf68: SuccessfulCreate - Created pod: agent-ops-ui-7f9fbf68-lq7cv
2026/09/06 18:01:42   17:59:11 INFO Deployment/notebooks-ui: ScalingReplicaSet - Scaled up replica set notebooks-ui-6577cc467 from 0 to 1
2026/09/06 18:01:42   17:59:11 INFO Pod/automl-ui-74d9b97d49-x68qc: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af"
2026/09/06 18:01:42   17:59:11 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-644d45d5b-xd7pt_9786af05-2e87-4c1b-ba58-8ad6c346e1fc became leader
2026/09/06 18:01:42   17:59:11 INFO Pod/automl-ui-74d9b97d49-x68qc: AddedInterface - Add eth0 [10.132.0.109/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:11 INFO ReplicaSet/notebooks-ui-6577cc467: SuccessfulCreate - Created pod: notebooks-ui-6577cc467-grs8s
2026/09/06 18:01:42   17:59:11 INFO ReplicaSet/maas-ui-74675d48f9: SuccessfulCreate - Created pod: maas-ui-74675d48f9-wjrcd
2026/09/06 18:01:42   17:59:11 WARN Pod/agent-ops-ui-7f9fbf68-lq7cv: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "agent-ops-proxy-tls" not found
2026/09/06 18:01:42   17:59:11 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: AddedInterface - Add eth0 [10.132.0.111/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:11 WARN Pod/maas-ui-74675d48f9-wjrcd: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "maas-proxy-tls" not found
2026/09/06 18:01:42   17:59:11 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987"
2026/09/06 18:01:42   17:59:11 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-7f9fbf68 from 0 to 1
2026/09/06 18:01:42   17:59:10 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-7c7dbc9d78-rwjvr to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:10 INFO Pod/autorag-ui-58f548cbd-z4g4s: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-58f548cbd-z4g4s to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:10 INFO ReplicaSet/autorag-ui-58f548cbd: SuccessfulCreate - Created pod: autorag-ui-58f548cbd-z4g4s
2026/09/06 18:01:42   17:59:10 INFO ReplicaSet/gen-ai-ui-7c7dbc9d78: SuccessfulCreate - Created pod: gen-ai-ui-7c7dbc9d78-rwjvr
2026/09/06 18:01:42   17:59:10 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-7c7dbc9d78 from 0 to 1
2026/09/06 18:01:42   17:59:10 INFO Pod/autorag-ui-58f548cbd-z4g4s: AddedInterface - Add eth0 [10.132.0.110/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:10 WARN Pod/automl-ui-74d9b97d49-x68qc: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/06 18:01:42   17:59:10 INFO Lease/opendatahub-feast-operator-lock: LeaderElection - opendatahub-feast-operator-84ff599545-j27hv_46af64a1-406c-4d38-8bc4-dafbe3bbeed8 became leader
2026/09/06 18:01:42   17:59:10 INFO Pod/autorag-ui-58f548cbd-z4g4s: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2"
2026/09/06 18:01:42   17:59:10 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-89f69845b-bbn56_7121972a-7846-4ba0-aec3-e2dc35aaea9f became leader
2026/09/06 18:01:42   17:59:10 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-58f548cbd from 0 to 1
2026/09/06 18:01:42   17:59:09 INFO Pod/automl-ui-74d9b97d49-x68qc: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-74d9b97d49-x68qc to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Created - Created container: core-bff
2026/09/06 18:01:42   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 534ms (534ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:42   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Started - Started container core-bff
2026/09/06 18:01:42   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 515ms (515ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:42   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Created - Created container: core-bff
2026/09/06 18:01:42   17:59:09 INFO ReplicaSet/automl-ui-74d9b97d49: SuccessfulCreate - Created pod: automl-ui-74d9b97d49-x68qc
2026/09/06 18:01:42   17:59:09 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-74d9b97d49 from 0 to 1
2026/09/06 18:01:42   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Started - Started container core-bff
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Created - Created container: rhods-dashboard
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Started - Started container kube-rbac-proxy
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Started - Started container rhods-dashboard
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: AddedInterface - Add eth0 [10.132.0.108/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Started - Started container kube-rbac-proxy
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Created - Created container: rhods-dashboard
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 395ms (395ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: AddedInterface - Add eth0 [10.133.0.110/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Started - Started container rhods-dashboard
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 330ms (330ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:42   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:42   17:59:07 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-797b4b4bd7-9c6vt to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:59:07 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-797b4b4bd7-4k4st to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:59:07 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:07 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:07 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulCreate - Created pod: rhods-dashboard-797b4b4bd7-9c6vt
2026/09/06 18:01:42   17:59:07 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/06 18:01:42   17:59:07 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulCreate - Created pod: rhods-dashboard-797b4b4bd7-4k4st
2026/09/06 18:01:42   17:59:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-797b4b4bd7 from 0 to 2
2026/09/06 18:01:42   17:59:06 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:06 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:04 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:59:03 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-669d6564bd-52hrd_df46cbe0-118c-4584-8cd8-6814c81a45f3 became leader
2026/09/06 18:01:42   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: AddedInterface - Add eth0 [10.134.0.78/23] from ovn-kubernetes
2026/09/06 18:01:42   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/06 18:01:42   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Started - Started container manager
2026/09/06 18:01:42   17:59:02 WARN Pod/rhods-dashboard-78d9976d4f-vpn4k: Unhealthy - Readiness probe failed: Get "https://10.133.0.58:8444/healthz": dial tcp 10.133.0.58:8444: connect: connection refused
2026/09/06 18:01:42   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Created - Created container: manager
2026/09/06 18:01:42   17:59:01 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f896225w to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:59:01 WARN Pod/rhods-dashboard-78d9976d4f-lcrbf: Unhealthy - Readiness probe failed: Get "https://10.132.0.51:8444/healthz": dial tcp 10.132.0.51:8444: connect: connection refused
2026/09/06 18:01:42   17:59:01 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 1 to 2
2026/09/06 18:01:42   17:59:01 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f896225w
2026/09/06 18:01:42   17:59:01 WARN Pod/rhods-dashboard-78d9976d4f-lcrbf: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:42   17:59:01 WARN Pod/rhods-dashboard-78d9976d4f-lcrbf: Unhealthy - Readiness probe failed: Get "https://10.132.0.51:8943/healthcheck": dial tcp 10.132.0.51:8943: connect: connection refused
2026/09/06 18:01:42   17:59:00 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7_f684a436-d1c1-4d7e-870c-6431ac18d567 became leader
2026/09/06 18:01:42   17:58:59 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:59 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:59 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Created - Created container: manager
2026/09/06 18:01:42   17:58:59 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Started - Started container manager
2026/09/06 18:01:42   17:58:59 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728" in 2.718s (2.718s including waiting). Image size: 186574763 bytes.
2026/09/06 18:01:42   17:58:56 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:56 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:56 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:58:56 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728"
2026/09/06 18:01:42   17:58:56 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-5f798b84bf from 0 to 1
2026/09/06 18:01:42   17:58:56 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: AddedInterface - Add eth0 [10.132.0.107/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:56 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-5f798b84bf: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7
2026/09/06 18:01:42   17:58:55 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810" in 1.634s (1.634s including waiting). Image size: 173568461 bytes.
2026/09/06 18:01:42   17:58:55 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Started - Started container manager
2026/09/06 18:01:42   17:58:55 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Created - Created container: manager
2026/09/06 18:01:42   17:58:54 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845" in 1.313s (1.313s including waiting). Image size: 168389130 bytes.
2026/09/06 18:01:42   17:58:54 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Started - Started container manager
2026/09/06 18:01:42   17:58:54 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Created - Created container: manager
2026/09/06 18:01:42   17:58:53 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845"
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Started - Started container manager
2026/09/06 18:01:42   17:58:53 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: AddedInterface - Add eth0 [10.134.0.77/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Started - Started container manager
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Created - Created container: manager
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 324ms (324ms including waiting). Image size: 209286205 bytes.
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/06 18:01:42   17:58:53 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810"
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 304ms (304ms including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:42   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Created - Created container: manager
2026/09/06 18:01:42   17:58:52 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-7654fd8d4b-m85xx to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:52 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75c8689979-2s8r8 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Started - Started container copy-manifests
2026/09/06 18:01:42   17:58:52 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75c8689979 from 0 to 1
2026/09/06 18:01:42   17:58:52 INFO ReplicaSet/notebook-controller-deployment-75c8689979: SuccessfulCreate - Created pod: notebook-controller-deployment-75c8689979-2s8r8
2026/09/06 18:01:42   17:58:52 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: AddedInterface - Add eth0 [10.133.0.109/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:52 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/06 18:01:42   17:58:52 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Created - Created container: manager
2026/09/06 18:01:42   17:58:52 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Started - Started container manager
2026/09/06 18:01:42   17:58:52 INFO ReplicaSet/odh-notebook-controller-manager-7654fd8d4b: SuccessfulCreate - Created pod: odh-notebook-controller-manager-7654fd8d4b-m85xx
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Started - Started container manager
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 327ms (327ms including waiting). Image size: 207018016 bytes.
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Started - Started container copy-manifests
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Created - Created container: copy-manifests
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 314ms (314ms including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Created - Created container: copy-manifests
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 321ms (321ms including waiting). Image size: 207018016 bytes.
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:42   17:58:52 WARN Pod/rhods-dashboard-78d9976d4f-vpn4k: Unhealthy - Readiness probe failed: Get "https://10.133.0.58:8943/healthcheck": dial tcp 10.133.0.58:8943: connect: connection refused
2026/09/06 18:01:42   17:58:52 WARN Pod/rhods-dashboard-78d9976d4f-vpn4k: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: AddedInterface - Add eth0 [10.132.0.106/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Started - Started container copy-manifests
2026/09/06 18:01:42   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Created - Created container: manager
2026/09/06 18:01:42   17:58:52 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-7654fd8d4b from 0 to 1
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Created - Created container: copy-manifests
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: AddedInterface - Add eth0 [10.133.0.108/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 352ms (352ms including waiting). Image size: 209286205 bytes.
2026/09/06 18:01:42   17:58:51 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-xd7pt to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:58:51 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-j27hv to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:51 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-q5z75 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:51 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-bbn56 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:51 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/06 18:01:42   17:58:51 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-bbn56
2026/09/06 18:01:42   17:58:51 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-q5z75
2026/09/06 18:01:42   17:58:51 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: AddedInterface - Add eth0 [10.134.0.76/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:51 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/06 18:01:42   17:58:51 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/06 18:01:42   17:58:51 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-xd7pt
2026/09/06 18:01:42   17:58:51 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/06 18:01:42   17:58:51 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-j27hv
2026/09/06 18:01:42   17:58:51 INFO Pod/ai-gateway-operator-89f69845b-bbn56: AddedInterface - Add eth0 [10.134.0.75/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:51 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/06 18:01:42   17:58:49 INFO Pod/workbenches-operator-c674d9c9-485g4: Created - Created container: manager
2026/09/06 18:01:42   17:58:49 INFO Pod/workbenches-operator-c674d9c9-485g4: Started - Started container manager
2026/09/06 18:01:42   17:58:49 INFO Pod/workbenches-operator-c674d9c9-485g4: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" already present on machine
2026/09/06 18:01:42   17:58:48 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Created - Created container: manager
2026/09/06 18:01:42   17:58:48 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:48 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:48 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4" in 1.188s (1.188s including waiting). Image size: 143076452 bytes.
2026/09/06 18:01:42   17:58:48 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7_647e62c2-b6e5-46ca-85e4-3e6dd5dfa3fe became leader
2026/09/06 18:01:42   17:58:48 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Started - Started container manager
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Created - Created container: manager
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 320ms (320ms including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:42   17:58:47 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: AddedInterface - Add eth0 [10.133.0.107/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:47 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Started - Started container manager
2026/09/06 18:01:42   17:58:47 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 2.723s (2.723s including waiting). Image size: 207018016 bytes.
2026/09/06 18:01:42   17:58:47 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Created - Created container: copy-manifests
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Started - Started container manager
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 1.827s (1.827s including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:42   17:58:47 INFO Pod/workbenches-operator-c674d9c9-485g4: Pulled - Successfully pulled image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" in 2.603s (2.603s including waiting). Image size: 185267738 bytes.
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Created - Created container: copy-manifests
2026/09/06 18:01:42   17:58:47 INFO Pod/dashboard-operator-669d6564bd-52hrd: Started - Started container manager
2026/09/06 18:01:42   17:58:47 INFO Pod/dashboard-operator-669d6564bd-52hrd: Created - Created container: manager
2026/09/06 18:01:42   17:58:47 INFO Pod/dashboard-operator-669d6564bd-52hrd: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" in 1.886s (1.886s including waiting). Image size: 170934264 bytes.
2026/09/06 18:01:42   17:58:47 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Started - Started container copy-manifests
2026/09/06 18:01:42   17:58:47 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4"
2026/09/06 18:01:42   17:58:47 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" in 2.658s (2.658s including waiting). Image size: 227888997 bytes.
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:42   17:58:47 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Created - Created container: manager
2026/09/06 18:01:42   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Started - Started container copy-manifests
2026/09/06 18:01:42   17:58:46 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:46 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-6b668bbbb: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7
2026/09/06 18:01:42   17:58:46 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-6b668bbbb from 0 to 1
2026/09/06 18:01:42   17:58:45 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: AddedInterface - Add eth0 [10.134.0.73/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Started - Started container manager
2026/09/06 18:01:42   17:58:45 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Pulling - Pulling image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b"
2026/09/06 18:01:42   17:58:45 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: AddedInterface - Add eth0 [10.134.0.72/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:45 INFO Pod/workbenches-operator-c674d9c9-485g4: Pulling - Pulling image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898"
2026/09/06 18:01:42   17:58:45 INFO Pod/dashboard-operator-669d6564bd-52hrd: AddedInterface - Add eth0 [10.133.0.105/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:45 INFO Pod/workbenches-operator-c674d9c9-485g4: AddedInterface - Add eth0 [10.134.0.74/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.726s (1.726s including waiting). Image size: 178395693 bytes.
2026/09/06 18:01:42   17:58:45 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:42   17:58:45 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: AddedInterface - Add eth0 [10.133.0.106/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:45 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/06 18:01:42   17:58:45 INFO Pod/dashboard-operator-669d6564bd-52hrd: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520"
2026/09/06 18:01:42   17:58:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Created - Created container: manager
2026/09/06 18:01:42   17:58:44 INFO Pod/workbenches-operator-c674d9c9-485g4: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-c674d9c9-485g4 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:44 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-gbbk4 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:44 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-zqdhr to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:44 INFO Pod/dashboard-operator-669d6564bd-52hrd: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-52hrd to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:44 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-7m6ch to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:44 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/06 18:01:42   17:58:44 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-gbbk4
2026/09/06 18:01:42   17:58:44 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/06 18:01:42   17:58:44 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 0 to 1
2026/09/06 18:01:42   17:58:44 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-7m6ch
2026/09/06 18:01:42   17:58:44 INFO ReplicaSet/workbenches-operator-c674d9c9: SuccessfulCreate - Created pod: workbenches-operator-c674d9c9-485g4
2026/09/06 18:01:42   17:58:44 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-52hrd
2026/09/06 18:01:42   17:58:44 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-c674d9c9 from 0 to 1
2026/09/06 18:01:42   17:58:44 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-zqdhr
2026/09/06 18:01:42   17:58:44 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/06 18:01:42   17:58:43 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89qqgjs to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:42   17:58:43 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/06 18:01:42   17:58:43 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89qqgjs
2026/09/06 18:01:42   17:58:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/06 18:01:42   17:58:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: AddedInterface - Add eth0 [10.132.0.105/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-76d74cfc9f-m4gjb to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-76d74cfc9f-9zbjx to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Started - Started container nginx
2026/09/06 18:01:42   17:58:42 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Started - Started container manager
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: AddedInterface - Add eth0 [10.134.0.71/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Created - Created container: nginx
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: AddedInterface - Add eth0 [10.133.0.104/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/06 18:01:42   17:58:42 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Pulled - Successfully pulled image "registry.k8s.io/pause:3.9" in 413ms (413ms including waiting). Image size: 750414 bytes.
2026/09/06 18:01:42   17:58:42 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Created - Created container: manager
2026/09/06 18:01:42   17:58:42 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-76d74cfc9f from 0 to 2
2026/09/06 18:01:42   17:58:42 INFO ReplicaSet/dashboard-redirect-76d74cfc9f: SuccessfulCreate - Created pod: dashboard-redirect-76d74cfc9f-m4gjb
2026/09/06 18:01:42   17:58:42 INFO ReplicaSet/dashboard-redirect-76d74cfc9f: SuccessfulCreate - Created pod: dashboard-redirect-76d74cfc9f-9zbjx
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Started - Started container nginx
2026/09/06 18:01:42   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Created - Created container: nginx
2026/09/06 18:01:42   17:58:41 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-6b8fb45c8b-mkpvh to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:42   17:58:41 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Pulling - Pulling image "registry.k8s.io/pause:3.9"
2026/09/06 18:01:42   17:58:41 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: AddedInterface - Add eth0 [10.133.0.103/23] from ovn-kubernetes
2026/09/06 18:01:42   17:58:41 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-6b8fb45c8b from 0 to 1
2026/09/06 18:01:42   17:58:41 INFO ReplicaSet/feast-operator-controller-manager-6b8fb45c8b: SuccessfulCreate - Created pod: feast-operator-controller-manager-6b8fb45c8b-mkpvh
2026/09/06 18:01:42   17:58:38 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:38 INFO Pod/odh-observability-59db79cb46-8nd4t: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:38 INFO Pod/dashboard-operator-669d6564bd-ft474: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:38 INFO Pod/aihub-controller-manager-d877888b8-8ncl2: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:37 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-q4hfc: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:37 INFO Pod/ai-gateway-operator-89f69845b-rd59r: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:37 INFO Pod/workbenches-operator-c674d9c9-gq8gm: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:37 INFO Pod/kserve-module-controller-manager-66ff7c977-xbgcj: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:36 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:36 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:36 INFO Pod/opendatahub-feast-operator-84ff599545-mqkgt: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:36 INFO Pod/opendatahub-ogx-operator-644d45d5b-ffzqf: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:36 INFO Pod/mlflow-operator-controller-manager-5bbb468f5f-8ww9r: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:36 INFO Pod/spark-operator-module-controller-manager-59549b5854-9ms94: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:33 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:33 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:33 INFO Pod/automl-ui-74d9b97d49-tz2vp: Killing - Stopping container automl-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/maas-ui-74675d48f9-fplwt: Killing - Stopping container maas-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/notebook-controller-deployment-75c8689979-5zrb8: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 INFO Pod/model-serving-api-7df8c9655f-zl6fj: Killing - Stopping container server
2026/09/06 18:01:42   17:58:33 INFO Pod/catalog-controller-manager-76985cd459-z9c28: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 INFO Pod/eval-hub-ui-869b89d9b4-x7vxb: Killing - Stopping container eval-hub-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/autorag-ui-58f548cbd-p7tvh: Killing - Stopping container autorag-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/agent-ops-ui-7f9fbf68-qlvdl: Killing - Stopping container agent-ops-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/model-registry-ui-7897899d9b-ls82g: Killing - Stopping container model-registry-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/odh-model-controller-6b9dd8b59d-4ssw4: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-s799q: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 INFO Pod/model-registry-operator-controller-manager-758fcb8c6b-c9wvs: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 INFO Pod/mlflow-ui-7bb6d46f5f-l56tj: Killing - Stopping container mlflow-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/odh-notebook-controller-manager-76dd46fbc-zl85r: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:33 INFO Pod/dashboard-redirect-76d74cfc9f-ldxqb: Killing - Stopping container nginx
2026/09/06 18:01:42   17:58:33 INFO Pod/notebooks-ui-6577cc467-55hxj: Killing - Stopping container notebooks-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/gen-ai-ui-7c7dbc9d78-jrt82: Killing - Stopping container gen-ai-ui
2026/09/06 18:01:42   17:58:33 INFO Pod/dashboard-redirect-76d74cfc9f-29zhn: Killing - Stopping container nginx
2026/09/06 18:01:42   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-lcrbf: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:42   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-lcrbf: Killing - Stopping container core-bff
2026/09/06 18:01:42   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-lcrbf: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:42   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-vpn4k: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:42   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-vpn4k: Killing - Stopping container core-bff
2026/09/06 18:01:42   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-vpn4k: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:42   17:58:32 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-tvvrq: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:32 INFO Pod/kserve-controller-manager-5fbbd6dd88-b5gv9: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4nthz: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-s8x8h: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:32 INFO Pod/maas-controller-56c898799-jxzbq: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:31 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:31 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:31 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:31 WARN Pod/spark-operator-webhook-5d466b7d66-bjfms: FailedToRetrieveImagePullSecret - Unable to retrieve some image pull secrets (spark-operator-webhook-dockercfg-ncltg); attempting to pull the image may not succeed.
2026/09/06 18:01:42   17:58:31 INFO Pod/spark-operator-controller-c594b967f-g9skp: Killing - Stopping container controller
2026/09/06 18:01:42   17:58:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89tmflj: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:31 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-j2w7b: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:31 INFO Pod/spark-operator-webhook-5d466b7d66-bjfms: Killing - Stopping container webhook
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:42   17:58:30 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-k2x48: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:30 INFO Pod/kserve-localmodelnode-agent-nvzxw: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:30 INFO Pod/kserve-localmodelnode-agent-7dkn9: Killing - Stopping container manager
2026/09/06 18:01:42   17:58:30 INFO Pod/kuberay-operator-7d87f54798-dkgpd: Killing - Stopping container kuberay-operator
2026/09/06 18:01:42   17:58:30 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-nvzxw
2026/09/06 18:01:42   17:58:30 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-7dkn9
2026/09/06 18:01:42 === RESOURCE QUOTAS ===
2026/09/06 18:01:42 No resource quotas found in monitored namespaces
2026/09/06 18:01:42 Classification for ConfigMap deletion recovery: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/06 18:01:42   Evidence: event Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8444/healthz": dial tcp 10.132.0.115:8444: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get \"https://10.132.0.115:8444/healthz\": dial tcp 10.132.0.115:8444: connect: connection refused"],"confidence":"medium"}
2026/09/06 18:01:42 Diagnostics complete for failed test: ConfigMap deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_well-known_LLMInferenceServiceConfig_versioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_subscription_dependency_conditions" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_platform_config_ConfigMap" classname="" time="0.030">
			<system-out><![CDATA[    helper_test.go:156: Verifying platform config ConfigMap odh-kserve-config exists in namespace redhat-ods-applications.
    helper_test.go:156: Non-XKS platform: verifying cert-manager CA keys are absent.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_ModelRegistry_state_propagation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery" classname="" time="241.730"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="20.290"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_inferenceservice-config" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-modelcache-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_odh-model-controller-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="45.660"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-controller-manager-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_localmodel-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_model-serving-api" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-webhook-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="175.780"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-controller-manager" classname="" time="35.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-localmodel-controller-manager" classname="" time="45.170"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_llmisvc-controller-manager" classname="" time="45.170"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_model-serving-api" classname="" time="25.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_odh-model-controller" classname="" time="25.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_disabled" classname="" time="10.380"></testcase>
		<testcase name="TestOdhOperator/components/group_2" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue" classname="" time="93.080"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry" classname="" time="70.640">
			<system-out><![CDATA[2026-09-06T18:06:15Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_enabled" classname="" time="50.260">
			<system-out><![CDATA[2026-09-06T18:06:15Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_error_with_ocp_kueue-operator_not_installed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_removed_to_unmanaged_transition" classname="" time="10.740">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Creating test namespace (test-kueue-managed-daeqn5s50gnc7mcd9cpg) with Kueue management annotation.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:156: Updating the management state of the component in the DataScienceCluster to Unmanaged with autoCreateQueues=true (expecting NotReady initially due to missing operator).
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Verifying the component in the DataScienceCluster transitions to Ready state.
    helper_test.go:156: Verifying Kueue configuration (cluster) is created with all expected frameworks.
    helper_test.go:156: Verifying default ClusterQueue and LocalQueue resources are created in namespace test-kueue-managed-daeqn5s50gnc7mcd9cpg.
    helper_test.go:156: Verifying default resource flavor (default-flavor) is created.
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind']]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_to_removed_transition" classname="" time="51.100">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Defining expected conditions for Unmanaged state (Ready=True).
    helper_test.go:156: Updating DSC default-dsc to set Kueue managementState to Unmanaged with autoCreateQueues=true.
    helper_test.go:156: Creating test namespace (test-kueue-managed-daeqn8k50gnc7mcd9cq0) with Kueue management annotation.
    helper_test.go:156: Verifying default Kueue resources exist in namespace test-kueue-managed-daeqn8k50gnc7mcd9cq0.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_module_CR_spec" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_registries_namespace_created" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_disabled" classname="" time="20.230">
			<system-out><![CDATA[    helper_test.go:156: Verifying default resource flavor (default-flavor) is created.
    helper_test.go:156: Updating DSC default-dsc to set Kueue managementState to Removed.
    helper_test.go:156: Verifying default resources are still present in namespace test-kueue-managed-daeqn8k50gnc7mcd9cq0.
    helper_test.go:156: Cleaning up Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_autoCreateQueues_disabled_skips_queue_creation" classname="" time="20.810">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No ConfigMap resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Service resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Deployment resources found for component Kueue, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_disabled" classname="" time="10.220">
			<system-out><![CDATA[    helper_test.go:156: Setting DSC component default-kueue to Removed state.
    helper_test.go:156: Verifying no Deployments remain in namespace redhat-ods-applications with part-of=kueue label.
    helper_test.go:156: Ensuring the component CR /default-kueue is removed.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator" classname="" time="91.350"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai" classname="" time="294.290">
			<system-out><![CDATA[2026-09-06T18:07:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_enabled" classname="" time="111.490">
			<system-out><![CDATA[2026-09-06T18:07:48Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_enabled" classname="" time="80.960"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_update_operand_resources" classname="" time="20.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_update_operand_resources/deployment_trustyai-service-operator-controller-manager" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_MCP_guardrails_mode" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_pre_check" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery" classname="" time="141.960"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="111.610"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-dsc-config" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-config" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-coding-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-instruction-following-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-leaderboard-v2" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-long-context-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-model-validation" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-open-telco-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-reasoning-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-safety-and-fairness-v1" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-standard-llm-evals-v1" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-collection-toxicity-and-ethical-principles" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-deepeval" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-garak" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-garak-kfp" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-guidellm" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-ibm-clear" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-inspect" classname="" time="5.120"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-lighteval" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-lm-evaluation-harness" classname="" time="5.140"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-evalhub-provider-ragas" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_trustyai-service-operator-lmeval-tasks" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="15.220"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-controller-manager-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_trustyai-service-operator-webhook-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.120"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_trustyai-service-operator-controller-manager" classname="" time="15.110"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_disabled" classname="" time="20.550"></testcase>
		<testcase name="TestOdhOperator/components/group_4" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve" classname="" time="101.480"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kueue" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:762: Skipping: flaky due to namespace stuck in Terminating state. See RHOAIENG-46773 and RHOAIENG-46774.]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer" classname="" time="40.410"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve/Validate_component_enabled" classname="" time="101.310">
			<system-out><![CDATA[2026-09-06T18:12:43Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer/Validate_component_enabled" classname="" time="40.240"></testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer/Validate_external_operator_degraded_condition_monitoring" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve/Validate_external_operator_degraded_condition_monitoring" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache" classname="" time="111.430">
			<system-out><![CDATA[2026-09-06T18:14:24Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_component_enabled" classname="" time="111.270"></testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_ConfigMap" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5/kserve-modelcache/Validate_ModelCache_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services" classname="" time="0.260"></testcase>
		<testcase name="TestOdhOperator/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/services/group_1/auth" classname="" time="0.260"></testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway" classname="" time="0.170">
			<system-out><![CDATA[2026-09-06T18:16:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_Auth_system_initialization" classname="" time="0.110">
			<system-out><![CDATA[2026-09-06T18:16:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_GatewayConfig_creation" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: Validating GatewayConfig resource
    helper_test.go:156: GatewayConfig validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_infrastructure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_client_and_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_proxy_secret_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_authentication_proxy_deployment" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_token_forwarding_to_dashboard" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OIDC_unauthenticated_access_redirects_to_login" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_kube-auth-proxy_TLS_args_match_cluster_APIServer" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_HorizontalPodAutoscaler_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_NetworkPolicy_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_OAuth_callback_HTTPRoute" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EnvoyFilter_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_EDS_endpoint_discovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_Gateway_ready_status" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: Validating Gateway ready status
    helper_test.go:156: Gateway ready status validation completed]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/gateway/Validate_dashboard_redirect_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_addition_of_RoleBinding_when_group_is_added" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_removal_of_bindings_when_a_group_is_removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_blocks_Auth_update_with_invalid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services/group_1/auth/Validate_CEL_allows_Auth_with_valid_groups" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests" classname="" time="0.140">
			<system-out><![CDATA[2026-09-06T18:16:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_operator_deployment_health" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_leader_election_behavior" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_success" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_components_deployment_failure" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_missing_CRD_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests/Validate_RBAC_restriction_handling" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/V2_to_V3_upgrade_E2E_Tests" classname="" time="0.140">
			<skipped message="Skipped"><![CDATA[2026-09-06T18:16:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests" classname="" time="0.150">
			<system-out><![CDATA[2026-09-06T18:16:16Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_creation_of_configmap_with_deletion_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests/Validate_that_owned_namespaces_are_not_deleted" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/upgrade_DSC_and_DSCI_v1_API" classname="" time="0.140">
			<skipped message="Skipped"><![CDATA[2026-09-06T18:16:17Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier3]]]></skipped>
		</testcase>
		<testcase name="TestLogPodsSectionContinuesWithPartialData" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesComponentsMatchRealTestGroup" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesServicesMatchRealTestGroup" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/space-separated_matches_the_real_contract" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/comma-separated_collapses_into_one_malformed_name_--_this_is_the_bug,_not_the_fix" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/single_name_has_no_separator_to_get_wrong" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/all_enabled_when_no_flags" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/explicit_inclusion_limits_to_named_tests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/exclusion_removes_from_full_set" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/inclusion_and_exclusion_combined" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test/monitoring" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:78: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:85: Test nonexistent is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:94: Test monitoring is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/auth" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1/gateway" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:128: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:142: No tests remaining in group single after excluding only]]></skipped>
		</testcase>
		<testcase name="TestIsTestEnabled" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/enabled_by_default" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/disabled_by_exclusion_flag" classname="" time="0.000"></testcase>
		<system-out><![CDATA[    controller_test.go:762: Restoring operator to original version for Phase 3
    controller_test.go:762: Patched Subscription to remove env vars: [RHAI_VERSION CI]
    controller_test.go:762: Waiting for DSC to fully reconcile at original version
    dag_ordering_test.go:193: Cleanup: setting all components to Removed, deleting quota, and restoring operator env vars
    dag_ordering_test.go:196: Patched Subscription to remove env vars: [RHAI_VERSION CI]
2026/09/06 18:01:42 TEST FAILURE DETECTED: ConfigMap deletion recovery
2026/09/06 18:01:42 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:ConfigMap deletion recovery) ===
2026/09/06 18:01:42 TEST FAILURE DETECTED: Validate resource deletion recovery
2026/09/06 18:01:42 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:Validate resource deletion recovery) ===
2026/09/06 18:01:43 === CLUSTER STATE ===
2026/09/06 18:01:43 Node: ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:01:43   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:01:43   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:01:43   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:01:43   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:01:43 Node: ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:01:43   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:01:43   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:01:43   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:01:43   Resources: 7500m CPU, 30892144Ki memory allocatable (of 8000m CPU, 32043120Ki memory capacity)
2026/09/06 18:01:43 Node: ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:01:43   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:01:43   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:01:43   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:01:43   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:01:43 === DEPLOYMENTS ===
2026/09/06 18:01:43 Failed to collect deployment data: deployments not ready: redhat-ods-applications/odh-model-controller (1/2)
2026/09/06 18:01:43 === PODS ===
2026/09/06 18:01:43 Failed to collect pod data: redhat-ods-applications/odh-model-controller-57457c4c89-44wfm: phase=Pending; redhat-ods-applications/rhods-dashboard-7b658c686f-29gd9: container kube-rbac-proxy not ready
2026/09/06 18:01:43 Continuing with partial pod data to collect logs from problematic containers
2026/09/06 18:01:43 Namespace: redhat-ods-applications
2026/09/06 18:01:43   Pod odh-model-controller-57457c4c89-44wfm: Pending
2026/09/06 18:01:43     Container manager: ready=false, waiting: ContainerCreating
2026/09/06 18:01:43   Pod rhods-dashboard-7b658c686f-29gd9: Running
2026/09/06 18:01:43     Container kube-rbac-proxy: ready=false
2026/09/06 18:01:43 Namespace kube-system: no problematic pods found
2026/09/06 18:01:43 === OPERATOR STATUS ===
2026/09/06 18:01:43 Operator deployment: 3/3 ready
2026/09/06 18:01:43 === DSCI STATUS ===
2026/09/06 18:01:43 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/06 18:01:43 === DSC STATUS ===
2026/09/06 18:01:43 DSC default-dsc:
2026/09/06 18:01:43   Ready: True - 
2026/09/06 18:01:43   ProvisioningSucceeded: True - 
2026/09/06 18:01:43   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/06 18:01:43   ModulesReady: True - 
2026/09/06 18:01:43   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:43   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:01:43   DashboardReady: True - 
2026/09/06 18:01:43   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   KserveLLMInferenceServiceDependencies: True - 
2026/09/06 18:01:43   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/06 18:01:43   KserveReady: True - 
2026/09/06 18:01:43   KueueReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:43   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:01:43   OGXReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   RayReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:43   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   TrainerReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/06 18:01:43   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/06 18:01:43   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/06 18:01:43 === RECENT EVENTS ===
2026/09/06 18:01:43   18:01:42 INFO Pod/odh-model-controller-57457c4c89-44wfm: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-44wfm to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   18:01:42 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-44wfm
2026/09/06 18:01:43   18:01:42 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/06 18:01:43   18:01:40 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8444/healthz": dial tcp 10.132.0.115:8444: connect: connection refused
2026/09/06 18:01:43   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container core-bff
2026/09/06 18:01:43   18:01:25 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7b658c686f from 1 to 0
2026/09/06 18:01:43   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Created - Created container: server
2026/09/06 18:01:43   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.56s (1.56s including waiting). Image size: 172459788 bytes.
2026/09/06 18:01:43   18:01:25 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:43   18:01:25 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8943/healthcheck": dial tcp 10.132.0.115:8943: connect: connection refused
2026/09/06 18:01:43   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Started - Started container server
2026/09/06 18:01:43   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:43   18:01:25 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulDelete - Deleted pod: rhods-dashboard-7b658c686f-29gd9
2026/09/06 18:01:43   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:43   18:01:23 INFO Pod/model-serving-api-7df8c9655f-4j78j: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/06 18:01:43   18:01:23 INFO Pod/model-serving-api-7df8c9655f-4j78j: AddedInterface - Add eth0 [10.132.0.127/23] from ovn-kubernetes
2026/09/06 18:01:43   18:01:22 INFO Pod/model-serving-api-7df8c9655f-4j78j: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-4j78j to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   18:01:22 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-4j78j
2026/09/06 18:01:43   18:01:22 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/06 18:01:43   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:43   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe failed: Get "https://10.133.0.110:8943/healthcheck": dial tcp 10.133.0.110:8943: connect: connection refused
2026/09/06 18:01:43   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe failed: Get "https://10.133.0.110:8444/healthz": dial tcp 10.133.0.110:8444: connect: connection refused
2026/09/06 18:01:43   18:01:08 INFO Pod/kuberay-operator-7d87f54798-s45kr: Killing - Stopping container kuberay-operator
2026/09/06 18:01:43   18:01:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Started - Started container manager
2026/09/06 18:01:43   18:01:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 2.257s (2.257s including waiting). Image size: 235279660 bytes.
2026/09/06 18:01:43   18:01:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Created - Created container: manager
2026/09/06 18:01:43   18:01:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: AddedInterface - Add eth0 [10.132.0.126/23] from ovn-kubernetes
2026/09/06 18:01:43   18:01:03 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:01:43   18:01:02 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-h588p to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   18:01:02 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-h588p
2026/09/06 18:01:43   18:01:02 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-s45kr to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Started - Started container kuberay-operator
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: AddedInterface - Add eth0 [10.132.0.125/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Killing - Stopping container kuberay-operator
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Killing - Stopping container kuberay-operator
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Created - Created container: kuberay-operator
2026/09/06 18:01:43   18:00:53 INFO Pod/kuberay-operator-7d87f54798-s45kr: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/06 18:01:43   18:00:53 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-s45kr
2026/09/06 18:01:43   18:00:53 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Started - Started container rhods-dashboard
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Started - Started container core-bff
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Created - Created container: core-bff
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 350ms (350ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Started - Started container kube-rbac-proxy
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Created - Created container: rhods-dashboard
2026/09/06 18:01:43   18:00:48 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 387ms (387ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:43   18:00:47 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-86c5fb98b4-wn46m to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   18:00:47 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Killing - Stopping container core-bff
2026/09/06 18:01:43   18:00:47 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: AddedInterface - Add eth0 [10.133.0.122/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:47 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-797b4b4bd7 from 1 to 0
2026/09/06 18:01:43   18:00:47 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-86c5fb98b4 from 1 to 2
2026/09/06 18:01:43   18:00:47 INFO ReplicaSet/rhods-dashboard-86c5fb98b4: SuccessfulCreate - Created pod: rhods-dashboard-86c5fb98b4-wn46m
2026/09/06 18:01:43   18:00:47 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:43   18:00:47 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulDelete - Deleted pod: rhods-dashboard-797b4b4bd7-9c6vt
2026/09/06 18:01:43   18:00:47 INFO Pod/rhods-dashboard-86c5fb98b4-wn46m: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:43   18:00:47 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:43   18:00:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Created - Created container: manager
2026/09/06 18:01:43   18:00:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Started - Started container manager
2026/09/06 18:01:43   18:00:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 2.063s (2.063s including waiting). Image size: 244249302 bytes.
2026/09/06 18:01:43   18:00:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: AddedInterface - Add eth0 [10.133.0.121/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:01:43   18:00:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-944q2 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   18:00:42 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-944q2
2026/09/06 18:01:43   18:00:42 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 1 to 2
2026/09/06 18:01:43   18:00:37 WARN Pod/rhods-dashboard-7b658c686f-wn6p5: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:43   18:00:37 WARN Pod/rhods-dashboard-7b658c686f-wn6p5: Unhealthy - Readiness probe failed: Get "https://10.133.0.118:8943/healthcheck": dial tcp 10.133.0.118:8943: connect: connection refused
2026/09/06 18:01:43   18:00:34 WARN Pod/rhods-dashboard-7b658c686f-wn6p5: Unhealthy - Readiness probe failed: Get "https://10.133.0.118:8444/healthz": dial tcp 10.133.0.118:8444: connect: connection refused
2026/09/06 18:01:43   18:00:25 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 2.402s (2.402s including waiting). Image size: 247667445 bytes.
2026/09/06 18:01:43   18:00:25 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Created - Created container: manager
2026/09/06 18:01:43   18:00:25 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Started - Started container manager
2026/09/06 18:01:43   18:00:23 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:01:43   18:00:23 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: AddedInterface - Add eth0 [10.134.0.83/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:23 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Killing - Stopping container manager
2026/09/06 18:01:43   18:00:22 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-h65m7 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   18:00:22 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 1 to 2
2026/09/06 18:01:43   18:00:22 WARN ReplicaSet/spark-operator-webhook-5d466b7d66: FailedCreate - Error creating: pods "spark-operator-webhook-5d466b7d66-" is forbidden: error looking up service account redhat-ods-applications/spark-operator-webhook: serviceaccount "spark-operator-webhook" not found
2026/09/06 18:01:43   18:00:22 WARN Pod/rhods-dashboard-797b4b4bd7-4k4st: Unhealthy - Readiness probe failed: Get "https://10.132.0.108:8444/healthz": dial tcp 10.132.0.108:8444: connect: connection refused
2026/09/06 18:01:43   18:00:22 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Killing - Stopping container webhook
2026/09/06 18:01:43   18:00:22 INFO Lease/spark-operator-controller-lock: LeaderElection - spark-operator-controller-c594b967f-mx9ls_69e073cb-90ee-454b-9975-f067656c5991 became leader
2026/09/06 18:01:43   18:00:22 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/06 18:01:43   18:00:22 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Killing - Stopping container controller
2026/09/06 18:01:43   18:00:22 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-h65m7
2026/09/06 18:01:43   18:00:21 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-5d466b7d66-zz4s5_44409d15-d15a-495f-b780-470d2e1a509d became leader
2026/09/06 18:01:43   18:00:20 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Created - Created container: kuberay-operator
2026/09/06 18:01:43   18:00:20 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Started - Started container kuberay-operator
2026/09/06 18:01:43   18:00:19 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.498s (1.498s including waiting). Image size: 189730715 bytes.
2026/09/06 18:01:43   18:00:19 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7d87f54798-l5rl9_f041aff2-fca6-4e53-aa64-1cda3cabfc48 became leader
2026/09/06 18:01:43   18:00:18 INFO Pod/kuberay-operator-7d87f54798-wxzdq: AddedInterface - Add eth0 [10.134.0.82/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:18 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/06 18:01:43   18:00:17 INFO Pod/kuberay-operator-7d87f54798-wxzdq: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-wxzdq to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   18:00:17 WARN Pod/rhods-dashboard-797b4b4bd7-4k4st: Unhealthy - Readiness probe failed: Get "https://10.132.0.108:8943/healthcheck": dial tcp 10.132.0.108:8943: connect: connection refused
2026/09/06 18:01:43   18:00:17 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-wxzdq
2026/09/06 18:01:43   18:00:17 WARN Pod/rhods-dashboard-797b4b4bd7-4k4st: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:43   18:00:17 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 1 to 2
2026/09/06 18:01:43   18:00:09 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 542ms (542ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:43   18:00:09 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Started - Started container core-bff
2026/09/06 18:01:43   18:00:09 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Created - Created container: core-bff
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Started - Started container kube-rbac-proxy
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 377ms (377ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: AddedInterface - Add eth0 [10.132.0.123/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Started - Started container rhods-dashboard
2026/09/06 18:01:43   18:00:08 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Created - Created container: rhods-dashboard
2026/09/06 18:01:43   18:00:07 INFO Pod/rhods-dashboard-86c5fb98b4-lvzfm: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-86c5fb98b4-lvzfm to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   18:00:07 INFO ReplicaSet/rhods-dashboard-86c5fb98b4: SuccessfulCreate - Created pod: rhods-dashboard-86c5fb98b4-lvzfm
2026/09/06 18:01:43   18:00:07 INFO Pod/autorag-ui-58f548cbd-z4g4s: Killing - Stopping container autorag-ui
2026/09/06 18:01:43   18:00:07 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:43   18:00:07 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Killing - Stopping container core-bff
2026/09/06 18:01:43   18:00:07 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:43   18:00:07 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulDelete - Deleted pod: rhods-dashboard-7b658c686f-wn6p5
2026/09/06 18:01:43   18:00:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-86c5fb98b4 from 0 to 1
2026/09/06 18:01:43   18:00:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7b658c686f from 2 to 1
2026/09/06 18:01:43   18:00:07 INFO Pod/automl-ui-74d9b97d49-x68qc: Killing - Stopping container automl-ui
2026/09/06 18:01:43   18:00:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   18:00:04 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-webhook-5d466b7d66-zz4s5 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-controller-c594b967f-mx9ls to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   18:00:04 INFO ReplicaSet/spark-operator-controller-c594b967f: SuccessfulCreate - Created pod: spark-operator-controller-c594b967f-mx9ls
2026/09/06 18:01:43   18:00:04 INFO ReplicaSet/spark-operator-webhook-5d466b7d66: SuccessfulCreate - Created pod: spark-operator-webhook-5d466b7d66-zz4s5
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Started - Started container webhook
2026/09/06 18:01:43   18:00:04 INFO Deployment/spark-operator-controller: ScalingReplicaSet - Scaled up replica set spark-operator-controller-c594b967f from 0 to 1
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Started - Started container controller
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Created - Created container: controller
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: AddedInterface - Add eth0 [10.133.0.120/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: AddedInterface - Add eth0 [10.133.0.119/23] from ovn-kubernetes
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Created - Created container: webhook
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-webhook-5d466b7d66-zz4s5: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/06 18:01:43   18:00:04 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Killing - Stopping container manager
2026/09/06 18:01:43   18:00:04 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/06 18:01:43   18:00:04 INFO Pod/spark-operator-controller-c594b967f-mx9ls: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/06 18:01:43   18:00:03 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   18:00:03 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   18:00:03 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   18:00:03 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   18:00:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Killing - Stopping container manager
2026/09/06 18:01:43   18:00:01 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-drj8q_49a3cad5-9238-4bb4-853f-095d4e050c85 became leader
2026/09/06 18:01:43   18:00:00 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.908s (1.908s including waiting). Image size: 189730715 bytes.
2026/09/06 18:01:43   18:00:00 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Created - Created container: kuberay-operator
2026/09/06 18:01:43   18:00:00 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-rtg5q_5ed5f5a4-56d7-43a3-b972-c60b4ae9b685 became leader
2026/09/06 18:01:43   18:00:00 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Started - Started container kuberay-operator
2026/09/06 18:01:43   17:59:59 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-jdmf2_4cdcf91a-008a-4a24-b6d1-c3b3814a8d19 became leader
2026/09/06 18:01:43   17:59:58 INFO Pod/kuberay-operator-7d87f54798-l5rl9: AddedInterface - Add eth0 [10.132.0.122/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:58 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/06 18:01:43   17:59:57 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:57 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:57 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Started - Started container core-bff
2026/09/06 18:01:43   17:59:57 WARN Pod/kuberay-operator-7d87f54798-l5rl9: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/06 18:01:43   17:59:57 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Created - Created container: core-bff
2026/09/06 18:01:43   17:59:57 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:57 INFO Lease/spark-operator-module-controller-manager: LeaderElection - spark-operator-module-controller-manager-59549b5854-69mlg_2ef4d6ec-bc98-48df-997a-6715d3827d7a became leader
2026/09/06 18:01:43   17:59:57 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 274ms (274ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:43   17:59:56 INFO Pod/kuberay-operator-7d87f54798-l5rl9: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-l5rl9 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Started - Started container rhods-dashboard
2026/09/06 18:01:43   17:59:56 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 289ms (289ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: AddedInterface - Add eth0 [10.133.0.118/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:56 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-l5rl9
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Started - Started container kube-rbac-proxy
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:43   17:59:56 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Created - Created container: rhods-dashboard
2026/09/06 18:01:43   17:59:55 INFO Pod/rhods-dashboard-7b658c686f-wn6p5: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7b658c686f-wn6p5 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:55 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7b658c686f from 1 to 2
2026/09/06 18:01:43   17:59:55 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-797b4b4bd7 from 2 to 1
2026/09/06 18:01:43   17:59:55 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-bpwt8_f7565ff5-f964-4439-8661-a4823fd2bbaf became leader
2026/09/06 18:01:43   17:59:55 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:43   17:59:55 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulCreate - Created pod: rhods-dashboard-7b658c686f-wn6p5
2026/09/06 18:01:43   17:59:55 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Killing - Stopping container core-bff
2026/09/06 18:01:43   17:59:55 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulDelete - Deleted pod: rhods-dashboard-797b4b4bd7-4k4st
2026/09/06 18:01:43   17:59:55 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:43   17:59:51 INFO Pod/kserve-localmodelnode-agent-mmmpl: Created - Created container: manager
2026/09/06 18:01:43   17:59:51 INFO Pod/kserve-localmodelnode-agent-nh54p: Started - Started container manager
2026/09/06 18:01:43   17:59:51 INFO Pod/kserve-localmodelnode-agent-nh54p: Created - Created container: manager
2026/09/06 18:01:43   17:59:51 INFO Pod/kserve-localmodelnode-agent-nh54p: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 436ms (436ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:01:43   17:59:51 INFO Pod/kserve-localmodelnode-agent-mmmpl: Started - Started container manager
2026/09/06 18:01:43   17:59:51 INFO Pod/kserve-localmodelnode-agent-mmmpl: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 447ms (447ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:01:43   17:59:50 INFO Pod/kserve-localmodelnode-agent-mmmpl: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-mmmpl to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:59:50 INFO Pod/kserve-localmodelnode-agent-nh54p: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-nh54p to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:50 INFO Pod/kserve-localmodelnode-agent-mmmpl: AddedInterface - Add eth0 [10.134.0.80/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:50 INFO Pod/kserve-localmodelnode-agent-mmmpl: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:01:43   17:59:50 INFO Pod/kserve-localmodelnode-agent-nh54p: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:01:43   17:59:50 INFO Pod/kserve-localmodelnode-agent-nh54p: AddedInterface - Add eth0 [10.133.0.117/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:44 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:43 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 540ms (540ms including waiting). Image size: 239132625 bytes.
2026/09/06 18:01:43   17:59:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/06 18:01:43   17:59:43 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Created - Created container: manager
2026/09/06 18:01:43   17:59:43 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Started - Started container manager
2026/09/06 18:01:43   17:59:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Started - Started container manager
2026/09/06 18:01:43   17:59:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Created - Created container: manager
2026/09/06 18:01:43   17:59:43 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Created - Created container: manager
2026/09/06 18:01:43   17:59:43 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 648ms (648ms including waiting). Image size: 244249302 bytes.
2026/09/06 18:01:43   17:59:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Started - Started container manager
2026/09/06 18:01:43   17:59:42 WARN Pod/kserve-localmodelnode-agent-mmmpl: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/06 18:01:43   17:59:42 WARN Pod/kserve-localmodelnode-agent-nh54p: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/06 18:01:43   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89xhn65 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:59:42 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:42 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2":
 Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:01:43   17:59:42 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "host
mount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:01:43   17:59:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-nh54p
2026/09/06 18:01:43   17:59:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-mmmpl
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: AddedInterface - Add eth0 [10.133.0.114/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89xhn65: AddedInterface - Add eth0 [10.134.0.79/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:01:43   17:59:42 INFO Pod/odh-model-controller-57457c4c89-rtg5q: AddedInterface - Add eth0 [10.132.0.121/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:42 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 390ms (390ms including waiting). Image size: 247667445 bytes.
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Created - Created container: manager
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Started - Started container manager
2026/09/06 18:01:43   17:59:42 INFO Pod/maas-controller-56c898799-m5mvz: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:42 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: AddedInterface - Add eth0 [10.132.0.120/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:42 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/06 18:01:43   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Started - Started container server
2026/09/06 18:01:43   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Created - Created container: server
2026/09/06 18:01:43   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/06 18:01:43   17:59:42 INFO Pod/model-serving-api-7df8c9655f-k4nbq: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:42 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89xhn65
2026/09/06 18:01:43   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Started - Started container manager
2026/09/06 18:01:43   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Created - Created container: manager
2026/09/06 18:01:43   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 370ms (370ms including waiting). Image size: 235279660 bytes.
2026/09/06 18:01:43   17:59:42 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:01:43   17:59:42 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:42 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:41 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:41 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-rtg5q to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:41 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-k4nbq to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:41 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-jdmf2 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:41 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-5g8dz to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:41 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-drj8q to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:41 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-5g8dz
2026/09/06 18:01:43   17:59:41 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/06 18:01:43   17:59:41 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-k4nbq
2026/09/06 18:01:43   17:59:41 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-jdmf2
2026/09/06 18:01:43   17:59:41 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/06 18:01:43   17:59:41 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-drj8q
2026/09/06 18:01:43   17:59:41 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-rtg5q
2026/09/06 18:01:43   17:59:41 INFO Lease/54e06e98.ogx.io: LeaderElection - ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt_ac428613-9176-4ff3-ace8-0d78242b4abc became leader
2026/09/06 18:01:43   17:59:41 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/06 18:01:43   17:59:41 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/06 18:01:43   17:59:41 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/06 18:01:43   17:59:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Started - Started container manager
2026/09/06 18:01:43   17:59:40 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Created - Created container: manager
2026/09/06 18:01:43   17:59:40 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Started - Started container manager
2026/09/06 18:01:43   17:59:39 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Pulled - Successfully pulled image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637" in 2.753s (2.753s including waiting). Image size: 191954597 bytes.
2026/09/06 18:01:43   17:59:39 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Pulled - Container image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" already present on machine
2026/09/06 18:01:43   17:59:39 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Created - Created container: manager
2026/09/06 18:01:43   17:59:38 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:38 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-bpwt8 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:38 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-bpwt8
2026/09/06 18:01:43   17:59:38 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:38 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-bpwt8: AddedInterface - Add eth0 [10.133.0.113/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:38 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/06 18:01:43   17:59:38 INFO Lease/maas-controller.models-as-a-service.opendatahub.io: LeaderElection - maas-controller-56c898799-m5mvz_82a9a590-d174-4f56-9ec0-3a18dd218dcc became leader
2026/09/06 18:01:43   17:59:37 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:37 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: AddedInterface - Add eth0 [10.133.0.112/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:37 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Pulling - Pulling image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637"
2026/09/06 18:01:43   17:59:36 INFO Pod/spark-operator-module-controller-manager-59549b5854-69mlg: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-module-controller-manager-59549b5854-69mlg to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:36 INFO Deployment/spark-operator-module-controller-manager: ScalingReplicaSet - Scaled up replica set spark-operator-module-controller-manager-59549b5854 from 0 to 1
2026/09/06 18:01:43   17:59:36 INFO ReplicaSet/spark-operator-module-controller-manager-59549b5854: SuccessfulCreate - Created pod: spark-operator-module-controller-manager-59549b5854-69mlg
2026/09/06 18:01:43   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-lgmzn to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: AddedInterface - Add eth0 [10.133.0.111/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:33 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-lgmzn
2026/09/06 18:01:43   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Started - Started container manager
2026/09/06 18:01:43   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Created - Created container: manager
2026/09/06 18:01:43   17:59:33 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-lgmzn: Pulled - Container image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" already present on machine
2026/09/06 18:01:43   17:59:33 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/06 18:01:43   17:59:27 INFO Pod/workbenches-operator-c674d9c9-485g4: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:25 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:25 WARN Pod/notebook-controller-deployment-75c8689979-2s8r8: Unhealthy - Readiness probe failed: Get "http://10.133.0.109:8081/readyz": dial tcp 10.133.0.109:8081: connect: connection refused
2026/09/06 18:01:43   17:59:25 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:24 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:22 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Created - Created container: manager
2026/09/06 18:01:43   17:59:22 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Started - Started container manager
2026/09/06 18:01:43   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: Started - Started container manager
2026/09/06 18:01:43   17:59:22 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Started - Started container manager
2026/09/06 18:01:43   17:59:22 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Created - Created container: manager
2026/09/06 18:01:43   17:59:22 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7" in 3.513s (3.513s including waiting). Image size: 184745870 bytes.
2026/09/06 18:01:43   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: Created - Created container: manager
2026/09/06 18:01:43   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" already present on machine
2026/09/06 18:01:43   17:59:22 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2" in 2.825s (2.825s including waiting). Image size: 193888070 bytes.
2026/09/06 18:01:43   17:59:22 INFO Pod/dashboard-operator-669d6564bd-zmng8: AddedInterface - Add eth0 [10.132.0.119/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:21 INFO Pod/dashboard-operator-669d6564bd-zmng8: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-zmng8 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:21 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:21 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 1 to 2
2026/09/06 18:01:43   17:59:21 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:21 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-zmng8
2026/09/06 18:01:43   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Started - Started container manager
2026/09/06 18:01:43   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Created - Created container: manager
2026/09/06 18:01:43   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd" in 662ms (662ms including waiting). Image size: 197531122 bytes.
2026/09/06 18:01:43   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd"
2026/09/06 18:01:43   17:59:19 INFO Pod/maas-controller-56c898799-m5mvz: AddedInterface - Add eth0 [10.132.0.116/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:19 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2"
2026/09/06 18:01:43   17:59:19 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: AddedInterface - Add eth0 [10.132.0.118/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:18 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:18 INFO ReplicaSet/ogx-k8s-operator-controller-manager-59777cdbbb: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt
2026/09/06 18:01:43   17:59:18 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Pulling - Pulling image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7"
2026/09/06 18:01:43   17:59:18 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: AddedInterface - Add eth0 [10.132.0.117/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:18 WARN Pod/ogx-k8s-operator-controller-manager-59777cdbbb-6gzpt: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "ogx-k8s-operator-webhook-cert" not found
2026/09/06 18:01:43   17:59:17 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-x498g: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-5fc6bb8d67-x498g to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:17 INFO Pod/maas-controller-56c898799-m5mvz: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-56c898799-m5mvz to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:17 INFO ReplicaSet/maas-controller-56c898799: SuccessfulCreate - Created pod: maas-controller-56c898799-m5mvz
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Started - Started container kube-rbac-proxy
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Created - Created container: rhods-dashboard
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Started - Started container rhods-dashboard
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:43   17:59:17 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-5fc6bb8d67 from 0 to 1
2026/09/06 18:01:43   17:59:17 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/06 18:01:43   17:59:17 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-59777cdbbb from 0 to 1
2026/09/06 18:01:43   17:59:17 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-56c898799 from 0 to 1
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 334ms (334ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:43   17:59:17 INFO ReplicaSet/feast-operator-controller-manager-5fc6bb8d67: SuccessfulCreate - Created pod: feast-operator-controller-manager-5fc6bb8d67-x498g
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Created - Created container: core-bff
2026/09/06 18:01:43   17:59:17 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Started - Started container core-bff
2026/09/06 18:01:43   17:59:17 WARN Pod/maas-controller-56c898799-m5mvz: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "maas-controller-webhook-cert" not found
2026/09/06 18:01:43   17:59:16 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 345ms (345ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:43   17:59:16 INFO Pod/maas-ui-74675d48f9-wjrcd: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe" in 3.918s (3.918s including waiting). Image size: 189581000 bytes.
2026/09/06 18:01:43   17:59:16 INFO Pod/rhods-dashboard-7b658c686f-29gd9: AddedInterface - Add eth0 [10.132.0.115/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:16 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987" in 4.949s (4.949s including waiting). Image size: 238860397 bytes.
2026/09/06 18:01:43   17:59:16 INFO Pod/maas-ui-74675d48f9-wjrcd: Started - Started container maas-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/maas-ui-74675d48f9-wjrcd: Created - Created container: maas-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/autorag-ui-58f548cbd-z4g4s: Started - Started container autorag-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Created - Created container: gen-ai-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:43   17:59:16 INFO Pod/autorag-ui-58f548cbd-z4g4s: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2" in 5.263s (5.264s including waiting). Image size: 223947455 bytes.
2026/09/06 18:01:43   17:59:16 INFO Pod/automl-ui-74d9b97d49-x68qc: Started - Started container automl-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/autorag-ui-58f548cbd-z4g4s: Created - Created container: autorag-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/notebooks-ui-6577cc467-grs8s: Created - Created container: notebooks-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Started - Started container gen-ai-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/notebooks-ui-6577cc467-grs8s: Started - Started container notebooks-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/notebooks-ui-6577cc467-grs8s: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003" in 3.772s (3.772s including waiting). Image size: 190593254 bytes.
2026/09/06 18:01:43   17:59:16 INFO Pod/automl-ui-74d9b97d49-x68qc: Created - Created container: automl-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Created - Created container: agent-ops-ui
2026/09/06 18:01:43   17:59:16 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3" in 3.579s (3.579s including waiting). Image size: 184646446 bytes.
2026/09/06 18:01:43   17:59:16 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Started - Started container agent-ops-ui
2026/09/06 18:01:43   17:59:15 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Killing - Stopping container manager
2026/09/06 18:01:43   17:59:15 INFO Pod/automl-ui-74d9b97d49-x68qc: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af" in 4.15s (4.15s including waiting). Image size: 222806700 bytes.
2026/09/06 18:01:43   17:59:13 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-q5z75_08d79772-5a7f-4b27-a7e8-ca0d73ebaaba became leader
2026/09/06 18:01:43   17:59:12 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-7b658c686f-29gd9 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:12 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3"
2026/09/06 18:01:43   17:59:12 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-7b658c686f from 0 to 1
2026/09/06 18:01:43   17:59:12 INFO Pod/maas-ui-74675d48f9-wjrcd: AddedInterface - Add eth0 [10.132.0.112/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:12 INFO Pod/maas-ui-74675d48f9-wjrcd: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe"
2026/09/06 18:01:43   17:59:12 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: AddedInterface - Add eth0 [10.132.0.113/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:12 INFO Pod/notebooks-ui-6577cc467-grs8s: AddedInterface - Add eth0 [10.132.0.114/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:12 INFO Pod/notebooks-ui-6577cc467-grs8s: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003"
2026/09/06 18:01:43   17:59:12 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulCreate - Created pod: rhods-dashboard-7b658c686f-29gd9
2026/09/06 18:01:43   17:59:11 INFO Pod/notebooks-ui-6577cc467-grs8s: Scheduled - Successfully assigned redhat-ods-applications/notebooks-ui-6577cc467-grs8s to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:11 INFO Pod/agent-ops-ui-7f9fbf68-lq7cv: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-7f9fbf68-lq7cv to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:11 INFO Pod/maas-ui-74675d48f9-wjrcd: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-74675d48f9-wjrcd to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:11 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-644d45d5b-xd7pt_9786af05-2e87-4c1b-ba58-8ad6c346e1fc became leader
2026/09/06 18:01:43   17:59:11 WARN Pod/agent-ops-ui-7f9fbf68-lq7cv: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "agent-ops-proxy-tls" not found
2026/09/06 18:01:43   17:59:11 INFO Pod/automl-ui-74d9b97d49-x68qc: AddedInterface - Add eth0 [10.132.0.109/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:11 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-7f9fbf68 from 0 to 1
2026/09/06 18:01:43   17:59:11 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-74675d48f9 from 0 to 1
2026/09/06 18:01:43   17:59:11 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: AddedInterface - Add eth0 [10.132.0.111/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:11 INFO ReplicaSet/notebooks-ui-6577cc467: SuccessfulCreate - Created pod: notebooks-ui-6577cc467-grs8s
2026/09/06 18:01:43   17:59:11 INFO Pod/automl-ui-74d9b97d49-x68qc: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af"
2026/09/06 18:01:43   17:59:11 WARN Pod/maas-ui-74675d48f9-wjrcd: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "maas-proxy-tls" not found
2026/09/06 18:01:43   17:59:11 INFO Deployment/notebooks-ui: ScalingReplicaSet - Scaled up replica set notebooks-ui-6577cc467 from 0 to 1
2026/09/06 18:01:43   17:59:11 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987"
2026/09/06 18:01:43   17:59:11 INFO ReplicaSet/maas-ui-74675d48f9: SuccessfulCreate - Created pod: maas-ui-74675d48f9-wjrcd
2026/09/06 18:01:43   17:59:11 INFO ReplicaSet/agent-ops-ui-7f9fbf68: SuccessfulCreate - Created pod: agent-ops-ui-7f9fbf68-lq7cv
2026/09/06 18:01:43   17:59:10 INFO Pod/gen-ai-ui-7c7dbc9d78-rwjvr: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-7c7dbc9d78-rwjvr to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:10 INFO Pod/autorag-ui-58f548cbd-z4g4s: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-58f548cbd-z4g4s to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:10 INFO Lease/opendatahub-feast-operator-lock: LeaderElection - opendatahub-feast-operator-84ff599545-j27hv_46af64a1-406c-4d38-8bc4-dafbe3bbeed8 became leader
2026/09/06 18:01:43   17:59:10 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-7c7dbc9d78 from 0 to 1
2026/09/06 18:01:43   17:59:10 INFO ReplicaSet/gen-ai-ui-7c7dbc9d78: SuccessfulCreate - Created pod: gen-ai-ui-7c7dbc9d78-rwjvr
2026/09/06 18:01:43   17:59:10 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-89f69845b-bbn56_7121972a-7846-4ba0-aec3-e2dc35aaea9f became leader
2026/09/06 18:01:43   17:59:10 INFO Pod/autorag-ui-58f548cbd-z4g4s: AddedInterface - Add eth0 [10.132.0.110/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:10 INFO Pod/autorag-ui-58f548cbd-z4g4s: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2"
2026/09/06 18:01:43   17:59:10 INFO ReplicaSet/autorag-ui-58f548cbd: SuccessfulCreate - Created pod: autorag-ui-58f548cbd-z4g4s
2026/09/06 18:01:43   17:59:10 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-58f548cbd from 0 to 1
2026/09/06 18:01:43   17:59:10 WARN Pod/automl-ui-74d9b97d49-x68qc: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "automl-proxy-tls" not found
2026/09/06 18:01:43   17:59:09 INFO Pod/automl-ui-74d9b97d49-x68qc: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-74d9b97d49-x68qc to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Created - Created container: core-bff
2026/09/06 18:01:43   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 534ms (534ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:43   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Started - Started container core-bff
2026/09/06 18:01:43   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 515ms (515ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:01:43   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Created - Created container: core-bff
2026/09/06 18:01:43   17:59:09 INFO ReplicaSet/automl-ui-74d9b97d49: SuccessfulCreate - Created pod: automl-ui-74d9b97d49-x68qc
2026/09/06 18:01:43   17:59:09 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-74d9b97d49 from 0 to 1
2026/09/06 18:01:43   17:59:09 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Started - Started container core-bff
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Created - Created container: rhods-dashboard
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Started - Started container kube-rbac-proxy
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Started - Started container rhods-dashboard
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: AddedInterface - Add eth0 [10.132.0.108/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Started - Started container kube-rbac-proxy
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Created - Created container: rhods-dashboard
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 395ms (395ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Created - Created container: kube-rbac-proxy
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: AddedInterface - Add eth0 [10.133.0.110/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Started - Started container rhods-dashboard
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 330ms (330ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:01:43   17:59:08 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:01:43   17:59:07 INFO Pod/rhods-dashboard-797b4b4bd7-9c6vt: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-797b4b4bd7-9c6vt to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:59:07 INFO Pod/rhods-dashboard-797b4b4bd7-4k4st: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-797b4b4bd7-4k4st to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:59:07 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:07 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:07 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulCreate - Created pod: rhods-dashboard-797b4b4bd7-9c6vt
2026/09/06 18:01:43   17:59:07 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/06 18:01:43   17:59:07 INFO ReplicaSet/rhods-dashboard-797b4b4bd7: SuccessfulCreate - Created pod: rhods-dashboard-797b4b4bd7-4k4st
2026/09/06 18:01:43   17:59:07 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-797b4b4bd7 from 0 to 2
2026/09/06 18:01:43   17:59:06 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:06 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:04 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:59:03 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-669d6564bd-52hrd_df46cbe0-118c-4584-8cd8-6814c81a45f3 became leader
2026/09/06 18:01:43   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: AddedInterface - Add eth0 [10.134.0.78/23] from ovn-kubernetes
2026/09/06 18:01:43   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/06 18:01:43   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Started - Started container manager
2026/09/06 18:01:43   17:59:02 WARN Pod/rhods-dashboard-78d9976d4f-vpn4k: Unhealthy - Readiness probe failed: Get "https://10.133.0.58:8444/healthz": dial tcp 10.133.0.58:8444: connect: connection refused
2026/09/06 18:01:43   17:59:02 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Created - Created container: manager
2026/09/06 18:01:43   17:59:01 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f896225w: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f896225w to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:59:01 WARN Pod/rhods-dashboard-78d9976d4f-lcrbf: Unhealthy - Readiness probe failed: Get "https://10.132.0.51:8444/healthz": dial tcp 10.132.0.51:8444: connect: connection refused
2026/09/06 18:01:43   17:59:01 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 1 to 2
2026/09/06 18:01:43   17:59:01 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f896225w
2026/09/06 18:01:43   17:59:01 WARN Pod/rhods-dashboard-78d9976d4f-lcrbf: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:43   17:59:01 WARN Pod/rhods-dashboard-78d9976d4f-lcrbf: Unhealthy - Readiness probe failed: Get "https://10.132.0.51:8943/healthcheck": dial tcp 10.132.0.51:8943: connect: connection refused
2026/09/06 18:01:43   17:59:00 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7_f684a436-d1c1-4d7e-870c-6431ac18d567 became leader
2026/09/06 18:01:43   17:58:59 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:59 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:59 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Created - Created container: manager
2026/09/06 18:01:43   17:58:59 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Started - Started container manager
2026/09/06 18:01:43   17:58:59 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728" in 2.718s (2.718s including waiting). Image size: 186574763 bytes.
2026/09/06 18:01:43   17:58:56 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:56 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:56 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7 to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:58:56 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728"
2026/09/06 18:01:43   17:58:56 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-5f798b84bf from 0 to 1
2026/09/06 18:01:43   17:58:56 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7: AddedInterface - Add eth0 [10.132.0.107/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:56 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-5f798b84bf: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-5f798b84bf-zgjh7
2026/09/06 18:01:43   17:58:55 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810" in 1.634s (1.634s including waiting). Image size: 173568461 bytes.
2026/09/06 18:01:43   17:58:55 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Started - Started container manager
2026/09/06 18:01:43   17:58:55 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Created - Created container: manager
2026/09/06 18:01:43   17:58:54 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845" in 1.313s (1.313s including waiting). Image size: 168389130 bytes.
2026/09/06 18:01:43   17:58:54 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Started - Started container manager
2026/09/06 18:01:43   17:58:54 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Created - Created container: manager
2026/09/06 18:01:43   17:58:53 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845"
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Started - Started container manager
2026/09/06 18:01:43   17:58:53 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: AddedInterface - Add eth0 [10.134.0.77/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Started - Started container manager
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Created - Created container: manager
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 324ms (324ms including waiting). Image size: 209286205 bytes.
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/06 18:01:43   17:58:53 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810"
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 304ms (304ms including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:43   17:58:53 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Created - Created container: manager
2026/09/06 18:01:43   17:58:52 INFO Pod/odh-notebook-controller-manager-7654fd8d4b-m85xx: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-7654fd8d4b-m85xx to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:52 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75c8689979-2s8r8 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Started - Started container copy-manifests
2026/09/06 18:01:43   17:58:52 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75c8689979 from 0 to 1
2026/09/06 18:01:43   17:58:52 INFO ReplicaSet/notebook-controller-deployment-75c8689979: SuccessfulCreate - Created pod: notebook-controller-deployment-75c8689979-2s8r8
2026/09/06 18:01:43   17:58:52 INFO Pod/notebook-controller-deployment-75c8689979-2s8r8: AddedInterface - Add eth0 [10.133.0.109/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:52 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/06 18:01:43   17:58:52 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Created - Created container: manager
2026/09/06 18:01:43   17:58:52 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Started - Started container manager
2026/09/06 18:01:43   17:58:52 INFO ReplicaSet/odh-notebook-controller-manager-7654fd8d4b: SuccessfulCreate - Created pod: odh-notebook-controller-manager-7654fd8d4b-m85xx
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Started - Started container manager
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 327ms (327ms including waiting). Image size: 207018016 bytes.
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Started - Started container copy-manifests
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Created - Created container: copy-manifests
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 314ms (314ms including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Created - Created container: copy-manifests
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 321ms (321ms including waiting). Image size: 207018016 bytes.
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:43   17:58:52 WARN Pod/rhods-dashboard-78d9976d4f-vpn4k: Unhealthy - Readiness probe failed: Get "https://10.133.0.58:8943/healthcheck": dial tcp 10.133.0.58:8943: connect: connection refused
2026/09/06 18:01:43   17:58:52 WARN Pod/rhods-dashboard-78d9976d4f-vpn4k: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: AddedInterface - Add eth0 [10.132.0.106/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Started - Started container copy-manifests
2026/09/06 18:01:43   17:58:52 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Created - Created container: manager
2026/09/06 18:01:43   17:58:52 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-7654fd8d4b from 0 to 1
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Created - Created container: copy-manifests
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: AddedInterface - Add eth0 [10.133.0.108/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:52 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 352ms (352ms including waiting). Image size: 209286205 bytes.
2026/09/06 18:01:43   17:58:51 INFO Pod/opendatahub-ogx-operator-644d45d5b-xd7pt: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-xd7pt to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:58:51 INFO Pod/opendatahub-feast-operator-84ff599545-j27hv: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-j27hv to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:51 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-q5z75 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:51 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-bbn56 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:51 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/06 18:01:43   17:58:51 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-bbn56
2026/09/06 18:01:43   17:58:51 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-q5z75
2026/09/06 18:01:43   17:58:51 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: AddedInterface - Add eth0 [10.134.0.76/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:51 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/06 18:01:43   17:58:51 INFO Pod/ai-gateway-operator-89f69845b-bbn56: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/06 18:01:43   17:58:51 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-xd7pt
2026/09/06 18:01:43   17:58:51 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/06 18:01:43   17:58:51 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-j27hv
2026/09/06 18:01:43   17:58:51 INFO Pod/ai-gateway-operator-89f69845b-bbn56: AddedInterface - Add eth0 [10.134.0.75/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:51 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/06 18:01:43   17:58:49 INFO Pod/workbenches-operator-c674d9c9-485g4: Created - Created container: manager
2026/09/06 18:01:43   17:58:49 INFO Pod/workbenches-operator-c674d9c9-485g4: Started - Started container manager
2026/09/06 18:01:43   17:58:49 INFO Pod/workbenches-operator-c674d9c9-485g4: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" already present on machine
2026/09/06 18:01:43   17:58:48 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Created - Created container: manager
2026/09/06 18:01:43   17:58:48 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:48 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:48 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4" in 1.188s (1.188s including waiting). Image size: 143076452 bytes.
2026/09/06 18:01:43   17:58:48 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7_647e62c2-b6e5-46ca-85e4-3e6dd5dfa3fe became leader
2026/09/06 18:01:43   17:58:48 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Started - Started container manager
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Created - Created container: manager
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 320ms (320ms including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:43   17:58:47 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: AddedInterface - Add eth0 [10.133.0.107/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:47 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Started - Started container manager
2026/09/06 18:01:43   17:58:47 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 2.723s (2.723s including waiting). Image size: 207018016 bytes.
2026/09/06 18:01:43   17:58:47 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Created - Created container: copy-manifests
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Started - Started container manager
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 1.827s (1.827s including waiting). Image size: 194988415 bytes.
2026/09/06 18:01:43   17:58:47 INFO Pod/workbenches-operator-c674d9c9-485g4: Pulled - Successfully pulled image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" in 2.603s (2.603s including waiting). Image size: 185267738 bytes.
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Created - Created container: copy-manifests
2026/09/06 18:01:43   17:58:47 INFO Pod/dashboard-operator-669d6564bd-52hrd: Started - Started container manager
2026/09/06 18:01:43   17:58:47 INFO Pod/dashboard-operator-669d6564bd-52hrd: Created - Created container: manager
2026/09/06 18:01:43   17:58:47 INFO Pod/dashboard-operator-669d6564bd-52hrd: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" in 1.886s (1.886s including waiting). Image size: 170934264 bytes.
2026/09/06 18:01:43   17:58:47 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Started - Started container copy-manifests
2026/09/06 18:01:43   17:58:47 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4"
2026/09/06 18:01:43   17:58:47 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" in 2.658s (2.658s including waiting). Image size: 227888997 bytes.
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:43   17:58:47 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Created - Created container: manager
2026/09/06 18:01:43   17:58:47 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Started - Started container copy-manifests
2026/09/06 18:01:43   17:58:46 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:46 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-6b668bbbb: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-6b668bbbbv5wv7
2026/09/06 18:01:43   17:58:46 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-6b668bbbb from 0 to 1
2026/09/06 18:01:43   17:58:45 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: AddedInterface - Add eth0 [10.134.0.73/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Started - Started container manager
2026/09/06 18:01:43   17:58:45 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Pulling - Pulling image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b"
2026/09/06 18:01:43   17:58:45 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: AddedInterface - Add eth0 [10.134.0.72/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:45 INFO Pod/workbenches-operator-c674d9c9-485g4: Pulling - Pulling image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898"
2026/09/06 18:01:43   17:58:45 INFO Pod/dashboard-operator-669d6564bd-52hrd: AddedInterface - Add eth0 [10.133.0.105/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:45 INFO Pod/workbenches-operator-c674d9c9-485g4: AddedInterface - Add eth0 [10.134.0.74/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.726s (1.726s including waiting). Image size: 178395693 bytes.
2026/09/06 18:01:43   17:58:45 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/06 18:01:43   17:58:45 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: AddedInterface - Add eth0 [10.133.0.106/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:45 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/06 18:01:43   17:58:45 INFO Pod/dashboard-operator-669d6564bd-52hrd: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520"
2026/09/06 18:01:43   17:58:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Created - Created container: manager
2026/09/06 18:01:43   17:58:44 INFO Pod/workbenches-operator-c674d9c9-485g4: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-c674d9c9-485g4 to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:44 INFO Pod/opendatahub-ogx-operator-644d45d5b-gbbk4: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-gbbk4 to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:44 INFO Pod/kserve-module-controller-manager-66ff7c977-zqdhr: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-zqdhr to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:44 INFO Pod/dashboard-operator-669d6564bd-52hrd: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-52hrd to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:44 INFO Pod/ai-gateway-operator-89f69845b-7m6ch: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-7m6ch to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:44 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/06 18:01:43   17:58:44 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-gbbk4
2026/09/06 18:01:43   17:58:44 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/06 18:01:43   17:58:44 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 0 to 1
2026/09/06 18:01:43   17:58:44 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-7m6ch
2026/09/06 18:01:43   17:58:44 INFO ReplicaSet/workbenches-operator-c674d9c9: SuccessfulCreate - Created pod: workbenches-operator-c674d9c9-485g4
2026/09/06 18:01:43   17:58:44 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-52hrd
2026/09/06 18:01:43   17:58:44 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-c674d9c9 from 0 to 1
2026/09/06 18:01:43   17:58:44 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-zqdhr
2026/09/06 18:01:43   17:58:44 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/06 18:01:43   17:58:43 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89qqgjs to ip-10-0-141-252.ec2.internal
2026/09/06 18:01:43   17:58:43 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/06 18:01:43   17:58:43 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89qqgjs
2026/09/06 18:01:43   17:58:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/06 18:01:43   17:58:43 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89qqgjs: AddedInterface - Add eth0 [10.132.0.105/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-76d74cfc9f-m4gjb to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-76d74cfc9f-9zbjx to ip-10-0-140-32.ec2.internal
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Started - Started container nginx
2026/09/06 18:01:43   17:58:42 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Started - Started container manager
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: AddedInterface - Add eth0 [10.134.0.71/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-9zbjx: Created - Created container: nginx
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: AddedInterface - Add eth0 [10.133.0.104/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/06 18:01:43   17:58:42 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Pulled - Successfully pulled image "registry.k8s.io/pause:3.9" in 413ms (413ms including waiting). Image size: 750414 bytes.
2026/09/06 18:01:43   17:58:42 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Created - Created container: manager
2026/09/06 18:01:43   17:58:42 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-76d74cfc9f from 0 to 2
2026/09/06 18:01:43   17:58:42 INFO ReplicaSet/dashboard-redirect-76d74cfc9f: SuccessfulCreate - Created pod: dashboard-redirect-76d74cfc9f-m4gjb
2026/09/06 18:01:43   17:58:42 INFO ReplicaSet/dashboard-redirect-76d74cfc9f: SuccessfulCreate - Created pod: dashboard-redirect-76d74cfc9f-9zbjx
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Started - Started container nginx
2026/09/06 18:01:43   17:58:42 INFO Pod/dashboard-redirect-76d74cfc9f-m4gjb: Created - Created container: nginx
2026/09/06 18:01:43   17:58:41 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-6b8fb45c8b-mkpvh to ip-10-0-134-105.ec2.internal
2026/09/06 18:01:43   17:58:41 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: Pulling - Pulling image "registry.k8s.io/pause:3.9"
2026/09/06 18:01:43   17:58:41 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-mkpvh: AddedInterface - Add eth0 [10.133.0.103/23] from ovn-kubernetes
2026/09/06 18:01:43   17:58:41 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-6b8fb45c8b from 0 to 1
2026/09/06 18:01:43   17:58:41 INFO ReplicaSet/feast-operator-controller-manager-6b8fb45c8b: SuccessfulCreate - Created pod: feast-operator-controller-manager-6b8fb45c8b-mkpvh
2026/09/06 18:01:43   17:58:38 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:38 INFO Pod/odh-observability-59db79cb46-8nd4t: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:38 INFO Pod/dashboard-operator-669d6564bd-ft474: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:38 INFO Pod/aihub-controller-manager-d877888b8-8ncl2: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:37 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-q4hfc: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:37 INFO Pod/ai-gateway-operator-89f69845b-rd59r: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:37 INFO Pod/workbenches-operator-c674d9c9-gq8gm: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:37 INFO Pod/kserve-module-controller-manager-66ff7c977-xbgcj: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:36 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:36 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:36 INFO Pod/opendatahub-feast-operator-84ff599545-mqkgt: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:36 INFO Pod/opendatahub-ogx-operator-644d45d5b-ffzqf: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:36 INFO Pod/mlflow-operator-controller-manager-5bbb468f5f-8ww9r: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:36 INFO Pod/spark-operator-module-controller-manager-59549b5854-9ms94: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:33 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:33 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:33 INFO Pod/automl-ui-74d9b97d49-tz2vp: Killing - Stopping container automl-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/maas-ui-74675d48f9-fplwt: Killing - Stopping container maas-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/notebook-controller-deployment-75c8689979-5zrb8: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 INFO Pod/model-serving-api-7df8c9655f-zl6fj: Killing - Stopping container server
2026/09/06 18:01:43   17:58:33 INFO Pod/catalog-controller-manager-76985cd459-z9c28: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 INFO Pod/eval-hub-ui-869b89d9b4-x7vxb: Killing - Stopping container eval-hub-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/autorag-ui-58f548cbd-p7tvh: Killing - Stopping container autorag-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/agent-ops-ui-7f9fbf68-qlvdl: Killing - Stopping container agent-ops-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/odh-model-controller-6b9dd8b59d-4ssw4: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 INFO Pod/model-registry-ui-7897899d9b-ls82g: Killing - Stopping container model-registry-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-s799q: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 INFO Pod/model-registry-operator-controller-manager-758fcb8c6b-c9wvs: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 INFO Pod/mlflow-ui-7bb6d46f5f-l56tj: Killing - Stopping container mlflow-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/odh-notebook-controller-manager-76dd46fbc-zl85r: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:33 INFO Pod/dashboard-redirect-76d74cfc9f-ldxqb: Killing - Stopping container nginx
2026/09/06 18:01:43   17:58:33 INFO Pod/notebooks-ui-6577cc467-55hxj: Killing - Stopping container notebooks-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/gen-ai-ui-7c7dbc9d78-jrt82: Killing - Stopping container gen-ai-ui
2026/09/06 18:01:43   17:58:33 INFO Pod/dashboard-redirect-76d74cfc9f-29zhn: Killing - Stopping container nginx
2026/09/06 18:01:43   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-lcrbf: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:43   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-lcrbf: Killing - Stopping container core-bff
2026/09/06 18:01:43   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-lcrbf: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:43   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-vpn4k: Killing - Stopping container rhods-dashboard
2026/09/06 18:01:43   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-vpn4k: Killing - Stopping container core-bff
2026/09/06 18:01:43   17:58:33 INFO Pod/rhods-dashboard-78d9976d4f-vpn4k: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:01:43   17:58:32 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-tvvrq: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:32 INFO Pod/kserve-controller-manager-5fbbd6dd88-b5gv9: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:32 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4nthz: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-s8x8h: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:32 INFO Pod/maas-controller-56c898799-jxzbq: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:31 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:31 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:31 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:31 WARN Pod/spark-operator-webhook-5d466b7d66-bjfms: FailedToRetrieveImagePullSecret - Unable to retrieve some image pull secrets (spark-operator-webhook-dockercfg-ncltg); attempting to pull the image may not succeed.
2026/09/06 18:01:43   17:58:31 INFO Pod/spark-operator-controller-c594b967f-g9skp: Killing - Stopping container controller
2026/09/06 18:01:43   17:58:31 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89tmflj: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:31 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-j2w7b: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:31 INFO Pod/spark-operator-webhook-5d466b7d66-bjfms: Killing - Stopping container webhook
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:01:43   17:58:30 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-k2x48: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:30 INFO Pod/kserve-localmodelnode-agent-nvzxw: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:30 INFO Pod/kserve-localmodelnode-agent-7dkn9: Killing - Stopping container manager
2026/09/06 18:01:43   17:58:30 INFO Pod/kuberay-operator-7d87f54798-dkgpd: Killing - Stopping container kuberay-operator
2026/09/06 18:01:43   17:58:30 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-nvzxw
2026/09/06 18:01:43   17:58:30 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-7dkn9
2026/09/06 18:01:43 === RESOURCE QUOTAS ===
2026/09/06 18:01:43 No resource quotas found in monitored namespaces
2026/09/06 18:01:43 Classification for Validate resource deletion recovery: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/06 18:01:43   Evidence: event Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8444/healthz": dial tcp 10.132.0.115:8444: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get \"https://10.132.0.115:8444/healthz\": dial tcp 10.132.0.115:8444: connect: connection refused"],"confidence":"medium"}
2026/09/06 18:01:43 Diagnostics complete for failed test: Validate resource deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/06 18:06:14 TEST FAILURE DETECTED: group 1
2026/09/06 18:06:14 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 1) ===
2026/09/06 18:06:15 === CLUSTER STATE ===
2026/09/06 18:06:15 Node: ip-10-0-134-105.ec2.internal
2026/09/06 18:06:15   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:06:15   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:06:15   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:06:15   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:06:15   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:06:15 Node: ip-10-0-140-32.ec2.internal
2026/09/06 18:06:15   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:06:15   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:06:15   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:06:15   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:06:15   Resources: 7500m CPU, 30892144Ki memory allocatable (of 8000m CPU, 32043120Ki memory capacity)
2026/09/06 18:06:15 Node: ip-10-0-141-252.ec2.internal
2026/09/06 18:06:15   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:06:15   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:06:15   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:06:15   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:06:15   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:06:15 === DEPLOYMENTS ===
2026/09/06 18:06:15 Namespace: redhat-ods-applications
2026/09/06 18:06:15   Deployment agent-ops-ui: 1/1 (READY)
2026/09/06 18:06:15   Deployment dashboard-operator: 2/2 (READY)
2026/09/06 18:06:15   Deployment dashboard-redirect: 2/2 (READY)
2026/09/06 18:06:15   Deployment gen-ai-ui: 1/1 (READY)
2026/09/06 18:06:15   Deployment maas-ui: 1/1 (READY)
2026/09/06 18:06:15   Deployment notebooks-ui: 1/1 (READY)
2026/09/06 18:06:15   Deployment rhods-dashboard: 2/2 (READY)
2026/09/06 18:06:15 Namespace: kube-system
2026/09/06 18:06:15 === PODS ===
2026/09/06 18:06:15 Namespace redhat-ods-applications: no problematic pods found
2026/09/06 18:06:15 Namespace kube-system: no problematic pods found
2026/09/06 18:06:15 === OPERATOR STATUS ===
2026/09/06 18:06:15 Operator deployment: 3/3 ready
2026/09/06 18:06:15 === DSCI STATUS ===
2026/09/06 18:06:15 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/06 18:06:15 === DSC STATUS ===
2026/09/06 18:06:15 DSC default-dsc:
2026/09/06 18:06:15   Ready: True - 
2026/09/06 18:06:15   ProvisioningSucceeded: True - 
2026/09/06 18:06:15   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/06 18:06:15   ModulesReady: True - 
2026/09/06 18:06:15   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/06 18:06:15   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:06:15   DashboardReady: True - 
2026/09/06 18:06:15   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   KserveLLMInferenceServiceDependencies: False - Submodule ManagementState is set to Removed
2026/09/06 18:06:15   KserveLLMInferenceServiceWideEPDependencies: False - Submodule ManagementState is set to Removed
2026/09/06 18:06:15   KserveReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   KueueReady: False - Component ManagementState is set to Removed
2026/09/06 18:06:15   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:06:15   OGXReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   RayReady: False - Component ManagementState is set to Removed
2026/09/06 18:06:15   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   TrainerReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/06 18:06:15   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/06 18:06:15   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/06 18:06:15 === RECENT EVENTS ===
2026/09/06 18:06:15   18:06:08 INFO Pod/kserve-module-controller-manager-66ff7c977-q5z75: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:05 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:06:15   18:06:05 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:06:15   18:06:05 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:05 INFO Pod/model-serving-api-7df8c9655f-fvt8x: Killing - Stopping container server
2026/09/06 18:06:15   18:06:05 INFO Pod/odh-model-controller-57457c4c89-tc7c5: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:05 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:05 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:04 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-mmmpl
2026/09/06 18:06:15   18:06:04 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-nh54p
2026/09/06 18:06:15   18:06:04 INFO Pod/kserve-localmodelnode-agent-nh54p: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:04 INFO Pod/kserve-localmodelnode-agent-mmmpl: Killing - Stopping container manager
2026/09/06 18:06:15   18:06:01 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-tc7c5_2a27605a-d0f6-4c3e-95bb-013ff9aaee93 became leader
2026/09/06 18:06:15   18:05:43 INFO Pod/odh-model-controller-57457c4c89-tc7c5: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 465ms (465ms including waiting). Image size: 239132625 bytes.
2026/09/06 18:06:15   18:05:43 INFO Pod/odh-model-controller-57457c4c89-tc7c5: Started - Started container manager
2026/09/06 18:06:15   18:05:43 INFO Pod/odh-model-controller-57457c4c89-tc7c5: Created - Created container: manager
2026/09/06 18:06:15   18:05:43 INFO Pod/odh-model-controller-57457c4c89-tc7c5: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/06 18:06:15   18:05:42 INFO Pod/odh-model-controller-57457c4c89-tc7c5: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-tc7c5 to ip-10-0-141-252.ec2.internal
2026/09/06 18:06:15   18:05:42 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/06 18:06:15   18:05:42 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-tc7c5
2026/09/06 18:06:15   18:05:42 INFO Pod/odh-model-controller-57457c4c89-tc7c5: AddedInterface - Add eth0 [10.132.0.131/23] from ovn-kubernetes
2026/09/06 18:06:15   18:05:39 INFO Pod/odh-model-controller-57457c4c89-rtg5q: Killing - Stopping container manager
2026/09/06 18:06:15   18:05:39 INFO Pod/odh-model-controller-57457c4c89-44wfm: Killing - Stopping container manager
2026/09/06 18:06:15   18:05:17 INFO Pod/model-serving-api-7df8c9655f-fvt8x: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-fvt8x to ip-10-0-141-252.ec2.internal
2026/09/06 18:06:15   18:05:17 INFO Pod/model-serving-api-7df8c9655f-fvt8x: Started - Started container server
2026/09/06 18:06:15   18:05:17 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-fvt8x
2026/09/06 18:06:15   18:05:17 INFO Pod/model-serving-api-7df8c9655f-fvt8x: Created - Created container: server
2026/09/06 18:06:15   18:05:17 INFO Pod/model-serving-api-7df8c9655f-fvt8x: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/06 18:06:15   18:05:17 INFO Pod/model-serving-api-7df8c9655f-fvt8x: AddedInterface - Add eth0 [10.132.0.130/23] from ovn-kubernetes
2026/09/06 18:06:15   18:05:17 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/06 18:06:15   18:05:14 INFO Pod/model-serving-api-7df8c9655f-k4nbq: Killing - Stopping container server
2026/09/06 18:06:15   18:05:14 INFO Pod/model-serving-api-7df8c9655f-4j78j: Killing - Stopping container server
2026/09/06 18:06:15   18:04:51 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-thn7r_70676364-d33a-451d-b7fe-8c06eb1cf047 became leader
2026/09/06 18:06:15   18:04:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 372ms (372ms including waiting). Image size: 235279660 bytes.
2026/09/06 18:06:15   18:04:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: Created - Created container: manager
2026/09/06 18:06:15   18:04:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: Started - Started container manager
2026/09/06 18:06:15   18:04:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-thn7r to ip-10-0-141-252.ec2.internal
2026/09/06 18:06:15   18:04:32 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/06 18:06:15   18:04:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: AddedInterface - Add eth0 [10.132.0.129/23] from ovn-kubernetes
2026/09/06 18:06:15   18:04:32 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-thn7r
2026/09/06 18:06:15   18:04:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-thn7r: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:06:15   18:04:29 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-h588p: Killing - Stopping container manager
2026/09/06 18:06:15   18:04:29 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-jdmf2: Killing - Stopping container manager
2026/09/06 18:06:15   18:03:48 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 384ms (384ms including waiting). Image size: 244249302 bytes.
2026/09/06 18:06:15   18:03:48 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: Started - Started container manager
2026/09/06 18:06:15   18:03:48 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: AddedInterface - Add eth0 [10.132.0.128/23] from ovn-kubernetes
2026/09/06 18:06:15   18:03:48 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:06:15   18:03:48 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: Created - Created container: manager
2026/09/06 18:06:15   18:03:47 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4trqs: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-4trqs to ip-10-0-141-252.ec2.internal
2026/09/06 18:06:15   18:03:47 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/06 18:06:15   18:03:47 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-4trqs
2026/09/06 18:06:15   18:03:44 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-944q2: Killing - Stopping container manager
2026/09/06 18:06:15   18:03:44 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-5g8dz: Killing - Stopping container manager
2026/09/06 18:06:15   18:03:27 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-72gfp_97be2c19-73c1-4a2e-adc7-13096aad8a20 became leader
2026/09/06 18:06:15   18:03:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 416ms (416ms including waiting). Image size: 247667445 bytes.
2026/09/06 18:06:15   18:03:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: Created - Created container: manager
2026/09/06 18:06:15   18:03:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:06:15   18:03:11 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: Started - Started container manager
2026/09/06 18:06:15   18:03:10 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-72gfp to ip-10-0-140-32.ec2.internal
2026/09/06 18:06:15   18:03:10 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/06 18:06:15   18:03:10 INFO Pod/kserve-controller-manager-5fbbd6dd88-72gfp: AddedInterface - Add eth0 [10.134.0.84/23] from ovn-kubernetes
2026/09/06 18:06:15   18:03:10 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-72gfp
2026/09/06 18:06:15   18:03:08 INFO Pod/kserve-controller-manager-5fbbd6dd88-h65m7: Killing - Stopping container manager
2026/09/06 18:06:15   18:03:08 INFO Pod/kserve-controller-manager-5fbbd6dd88-drj8q: Killing - Stopping container manager
2026/09/06 18:06:15   18:01:55 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = NotFound desc = container is not created or running: checking if PID of 9848a498f027fd54faa948eaa3cedbd52fabfe7d242313a6e89ba80665671bfb is running failed: container process not found
2026/09/06 18:06:15   18:01:55 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8943/healthcheck": dial tcp 10.132.0.115:8943: connect: connection refused
2026/09/06 18:06:15   18:01:55 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe failed: Get "https://10.132.0.115:8444/healthz": dial tcp 10.132.0.115:8444: connect: connection refused
2026/09/06 18:06:15   18:01:45 INFO Pod/odh-model-controller-57457c4c89-44wfm: Started - Started container manager
2026/09/06 18:06:15   18:01:45 INFO Pod/odh-model-controller-57457c4c89-44wfm: Created - Created container: manager
2026/09/06 18:06:15   18:01:45 INFO Pod/odh-model-controller-57457c4c89-44wfm: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 2.048s (2.049s including waiting). Image size: 239132625 bytes.
2026/09/06 18:06:15   18:01:43 INFO Pod/odh-model-controller-57457c4c89-44wfm: AddedInterface - Add eth0 [10.133.0.123/23] from ovn-kubernetes
2026/09/06 18:06:15   18:01:43 INFO Pod/odh-model-controller-57457c4c89-44wfm: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/06 18:06:15   18:01:42 INFO Pod/odh-model-controller-57457c4c89-44wfm: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-44wfm to ip-10-0-134-105.ec2.internal
2026/09/06 18:06:15   18:01:42 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-44wfm
2026/09/06 18:06:15   18:01:42 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/06 18:06:15   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.56s (1.56s including waiting). Image size: 172459788 bytes.
2026/09/06 18:06:15   18:01:25 WARN Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:06:15   18:01:25 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-7b658c686f from 1 to 0
2026/09/06 18:06:15   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Created - Created container: server
2026/09/06 18:06:15   18:01:25 INFO ReplicaSet/rhods-dashboard-7b658c686f: SuccessfulDelete - Deleted pod: rhods-dashboard-7b658c686f-29gd9
2026/09/06 18:06:15   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:06:15   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container rhods-dashboard
2026/09/06 18:06:15   18:01:25 INFO Pod/rhods-dashboard-7b658c686f-29gd9: Killing - Stopping container core-bff
2026/09/06 18:06:15   18:01:25 INFO Pod/model-serving-api-7df8c9655f-4j78j: Started - Started container server
2026/09/06 18:06:15   18:01:23 INFO Pod/model-serving-api-7df8c9655f-4j78j: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/06 18:06:15   18:01:23 INFO Pod/model-serving-api-7df8c9655f-4j78j: AddedInterface - Add eth0 [10.132.0.127/23] from ovn-kubernetes
2026/09/06 18:06:15   18:01:22 INFO Pod/model-serving-api-7df8c9655f-4j78j: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-4j78j to ip-10-0-141-252.ec2.internal
2026/09/06 18:06:15   18:01:22 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/06 18:06:15   18:01:22 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-4j78j
2026/09/06 18:06:15   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:06:15   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe failed: Get "https://10.133.0.110:8943/healthcheck": dial tcp 10.133.0.110:8943: connect: connection refused
2026/09/06 18:06:15   18:01:16 WARN Pod/rhods-dashboard-797b4b4bd7-9c6vt: Unhealthy - Readiness probe failed: Get "https://10.133.0.110:8444/healthz": dial tcp 10.133.0.110:8444: connect: connection refused
2026/09/06 18:06:15 === RESOURCE QUOTAS ===
2026/09/06 18:06:15 No resource quotas found in monitored namespaces
2026/09/06 18:06:15 Classification for group 1: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/06 18:06:15   Evidence: event Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = NotFound desc = container is not created or running: checking if PID of 9848a498f027fd54faa948eaa3cedbd52fabfe7d242313a6e89ba80665671bfb is running failed: container process not found
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-7b658c686f-29gd9: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = NotFound desc = container is not created or running: checking if PID of 9848a498f027fd54faa948eaa3cedbd52fabfe7d242313a6e89ba80665671bfb is running failed: container process not found"],"confidence":"medium"}
2026/09/06 18:06:15 Diagnostics complete for failed test: group 1
    controller_test.go:762: Stopping: group 1 test failed.
2026/09/06 18:06:15 Circuit breaker: failure classified as infrastructure/probe-failure (code=1014)
2026/09/06 18:06:15 Circuit breaker: infrastructure/unknown failure recorded (1/3)
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Creating test namespace (test-kueue-no-queues-daeqnlc50gnc7mcd9cqg) with Kueue management annotation.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:156: Updating DSC to set Kueue to Unmanaged with autoCreateQueues=false (default).
    helper_test.go:156: Verifying Kueue configuration (cluster) is still created.
    helper_test.go:156: Verifying no ClusterQueue, ResourceFlavor or LocalQueue is created (autoCreateQueues=false).
    helper_test.go:156: Cleaning up Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind'
2026-09-06T18:12:43Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
2026/09/06 18:16:15 TEST FAILURE DETECTED: components
2026/09/06 18:16:15 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:components) ===
2026/09/06 18:16:16 === CLUSTER STATE ===
2026/09/06 18:16:16 Node: ip-10-0-134-105.ec2.internal
2026/09/06 18:16:16   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:16:16   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:16:16   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:16:16   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:16:16   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:16:16 Node: ip-10-0-140-32.ec2.internal
2026/09/06 18:16:16   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:16:16   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:16:16   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:16:16   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:16:16   Resources: 7500m CPU, 30892144Ki memory allocatable (of 8000m CPU, 32043120Ki memory capacity)
2026/09/06 18:16:16 Node: ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/06 18:16:16   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/06 18:16:16   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/06 18:16:16   Condition Ready: True - kubelet is posting ready status
2026/09/06 18:16:16   Resources: 7500m CPU, 31236208Ki memory allocatable (of 8000m CPU, 32387184Ki memory capacity)
2026/09/06 18:16:16 === DEPLOYMENTS ===
2026/09/06 18:16:16 Namespace: redhat-ods-applications
2026/09/06 18:16:16   Deployment agent-ops-ui: 1/1 (READY)
2026/09/06 18:16:16   Deployment dashboard-operator: 2/2 (READY)
2026/09/06 18:16:16   Deployment dashboard-redirect: 2/2 (READY)
2026/09/06 18:16:16   Deployment gen-ai-ui: 1/1 (READY)
2026/09/06 18:16:16   Deployment kserve-controller-manager: 1/1 (READY)
2026/09/06 18:16:16   Deployment kserve-localmodel-controller-manager: 1/1 (READY)
2026/09/06 18:16:16   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/06 18:16:16   Deployment kubeflow-trainer-controller-manager: 1/1 (READY)
2026/09/06 18:16:16   Deployment llmisvc-controller-manager: 1/1 (READY)
2026/09/06 18:16:16   Deployment maas-ui: 1/1 (READY)
2026/09/06 18:16:16   Deployment model-serving-api: 1/1 (READY)
2026/09/06 18:16:16   Deployment notebooks-ui: 1/1 (READY)
2026/09/06 18:16:16   Deployment odh-model-controller: 1/1 (READY)
2026/09/06 18:16:16   Deployment rhods-dashboard: 2/2 (READY)
2026/09/06 18:16:16   Deployment trainer-operator-controller-manager: 1/1 (READY)
2026/09/06 18:16:16 Namespace: kube-system
2026/09/06 18:16:16 === PODS ===
2026/09/06 18:16:16 Namespace redhat-ods-applications: no problematic pods found
2026/09/06 18:16:16 Namespace kube-system: no problematic pods found
2026/09/06 18:16:16 === OPERATOR STATUS ===
2026/09/06 18:16:16 Operator deployment: 3/3 ready
2026/09/06 18:16:16 === DSCI STATUS ===
2026/09/06 18:16:16 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/06 18:16:16 === DSC STATUS ===
2026/09/06 18:16:16 DSC default-dsc:
2026/09/06 18:16:16   Ready: True - 
2026/09/06 18:16:16   ProvisioningSucceeded: True - 
2026/09/06 18:16:16   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/06 18:16:16   ModulesReady: True - 
2026/09/06 18:16:16   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/06 18:16:16   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:16:16   DashboardReady: True - 
2026/09/06 18:16:16   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   KserveLLMInferenceServiceDependencies: True - 
2026/09/06 18:16:16   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/06 18:16:16   KserveReady: True - 
2026/09/06 18:16:16   KueueReady: False - Component ManagementState is set to Removed
2026/09/06 18:16:16   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/06 18:16:16   OGXReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   RayReady: False - Component ManagementState is set to Removed
2026/09/06 18:16:16   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   TrainerReady: True - 
2026/09/06 18:16:16   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/06 18:16:16   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/06 18:16:16   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/06 18:16:16 === RECENT EVENTS ===
2026/09/06 18:16:16   18:15:49 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-5t9jr_c24b8739-e84f-44a2-b603-e65aa821f070 became leader
2026/09/06 18:16:16   18:15:47 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-fh525_03f56aff-39cf-424f-a603-58dc17972b3d became leader
2026/09/06 18:16:16   18:15:47 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-sqz4c_da2223c6-2ac8-4edc-9851-75a61981707c became leader
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-rw779: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-rw779: Created - Created container: manager
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-rw779: Started - Started container manager
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-s8gsw: Created - Created container: manager
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-rw779: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 456ms (456ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-s8gsw: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 515ms (515ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-s8gsw: Started - Started container manager
2026/09/06 18:16:16   18:15:36 INFO Pod/kserve-localmodelnode-agent-rw779: AddedInterface - Add eth0 [10.134.0.97/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:35 INFO Pod/kserve-localmodelnode-agent-s8gsw: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-s8gsw to ip-10-0-134-105.ec2.internal
2026/09/06 18:16:16   18:15:35 INFO Pod/kserve-localmodelnode-agent-rw779: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-rw779 to ip-10-0-140-32.ec2.internal
2026/09/06 18:16:16   18:15:35 INFO Pod/kserve-localmodelnode-agent-s8gsw: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:16:16   18:15:35 INFO Pod/kserve-localmodelnode-agent-s8gsw: AddedInterface - Add eth0 [10.133.0.132/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:31 INFO Pod/kserve-controller-manager-5fbbd6dd88-5t9jr: Started - Started container manager
2026/09/06 18:16:16   18:15:31 INFO Pod/kserve-controller-manager-5fbbd6dd88-5t9jr: Created - Created container: manager
2026/09/06 18:16:16   18:15:31 INFO Pod/kserve-controller-manager-5fbbd6dd88-5t9jr: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 2.33s (2.33s including waiting). Image size: 247667445 bytes.
2026/09/06 18:16:16   18:15:30 INFO Pod/odh-model-controller-57457c4c89-sqz4c: Started - Started container manager
2026/09/06 18:16:16   18:15:30 INFO Pod/odh-model-controller-57457c4c89-sqz4c: Created - Created container: manager
2026/09/06 18:16:16   18:15:30 INFO Pod/odh-model-controller-57457c4c89-sqz4c: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 365ms (365ms including waiting). Image size: 239132625 bytes.
2026/09/06 18:16:16   18:15:29 WARN Pod/kserve-localmodelnode-agent-rw779: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/06 18:16:16   18:15:29 WARN Pod/kserve-localmodelnode-agent-s8gsw: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/06 18:16:16   18:15:29 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-fh525: Started - Started container manager
2026/09/06 18:16:16   18:15:29 INFO Pod/odh-model-controller-57457c4c89-sqz4c: AddedInterface - Add eth0 [10.132.0.166/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:29 INFO Pod/model-serving-api-7df8c9655f-x97fd: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/06 18:16:16   18:15:29 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-fh525: Created - Created container: manager
2026/09/06 18:16:16   18:15:29 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-fh525: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 547ms (547ms including waiting). Image size: 235279660 bytes.
2026/09/06 18:16:16   18:15:29 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-s8gsw
2026/09/06 18:16:16   18:15:29 INFO Pod/model-serving-api-7df8c9655f-x97fd: Started - Started container server
2026/09/06 18:16:16   18:15:29 INFO Pod/model-serving-api-7df8c9655f-x97fd: Created - Created container: server
2026/09/06 18:16:16   18:15:29 INFO Pod/odh-model-controller-57457c4c89-sqz4c: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/06 18:16:16   18:15:29 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: AddedInterface - Add eth0 [10.132.0.163/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:29 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:16:16   18:15:29 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 353ms (353ms including waiting). Image size: 244249302 bytes.
2026/09/06 18:16:16   18:15:29 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: Created - Created container: manager
2026/09/06 18:16:16   18:15:29 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: Started - Started container manager
2026/09/06 18:16:16   18:15:29 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-rw779
2026/09/06 18:16:16   18:15:28 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:15:28 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:15:28 INFO Pod/odh-model-controller-57457c4c89-sqz4c: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-sqz4c to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:15:28 INFO Pod/model-serving-api-7df8c9655f-x97fd: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-x97fd to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:15:28 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-fh525: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-fh525 to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:15:28 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-x4s7n to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:15:28 INFO Pod/kserve-controller-manager-5fbbd6dd88-5t9jr: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-5t9jr to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:15:28 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-x97fd
2026/09/06 18:16:16   18:15:28 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-sqz4c
2026/09/06 18:16:16   18:15:28 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/06 18:16:16   18:15:28 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2":
 Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:16:16   18:15:28 WARN Pod/odh-model-controller-57457c4c89-sqz4c: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found
2026/09/06 18:16:16   18:15:28 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/06 18:16:16   18:15:28 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-x4s7n
2026/09/06 18:16:16   18:15:28 INFO Pod/model-serving-api-7df8c9655f-x97fd: AddedInterface - Add eth0 [10.132.0.165/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:28 WARN Pod/kserve-localmodel-controller-manager-fc8c879d7-x4s7n: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "localmodel-webhook-server-cert" not found
2026/09/06 18:16:16   18:15:28 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "openshift-ai-inferenceservice-image-volume-scc": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "openshift-ai-llminferenceservice-scc": Forbidden: not usable by user or serviceaccount, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable
 by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:16:16   18:15:28 INFO Pod/kserve-controller-manager-5fbbd6dd88-5t9jr: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:16:16   18:15:28 INFO Pod/kserve-controller-manager-5fbbd6dd88-5t9jr: AddedInterface - Add eth0 [10.132.0.162/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:28 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/06 18:16:16   18:15:28 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-fh525
2026/09/06 18:16:16   18:15:28 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-fh525: AddedInterface - Add eth0 [10.132.0.164/23] from ovn-kubernetes
2026/09/06 18:16:16   18:15:28 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/06 18:16:16   18:15:28 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-fh525: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:16:16   18:15:28 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/06 18:16:16   18:15:28 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-5t9jr
2026/09/06 18:16:16   18:15:01 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-cfw4g_4e5f7def-5399-4564-a1eb-309efda77825 became leader
2026/09/06 18:16:16   18:14:37 INFO Pod/kserve-module-controller-manager-66ff7c977-cfw4g: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-cfw4g to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:14:37 INFO Pod/kserve-module-controller-manager-66ff7c977-cfw4g: Pulled - Container image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" already present on machine
2026/09/06 18:16:16   18:14:37 INFO Pod/kserve-module-controller-manager-66ff7c977-cfw4g: Created - Created container: manager
2026/09/06 18:16:16   18:14:37 INFO Pod/kserve-module-controller-manager-66ff7c977-cfw4g: AddedInterface - Add eth0 [10.132.0.160/23] from ovn-kubernetes
2026/09/06 18:16:16   18:14:37 INFO Pod/kserve-module-controller-manager-66ff7c977-cfw4g: Started - Started container manager
2026/09/06 18:16:16   18:14:37 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-cfw4g
2026/09/06 18:16:16   18:14:36 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/06 18:16:16   18:14:30 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:29 INFO Pod/odh-model-controller-57457c4c89-wg65r: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:29 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:29 INFO Pod/model-serving-api-7df8c9655f-flhvp: Killing - Stopping container server
2026/09/06 18:16:16   18:14:29 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:29 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:28 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:14:28 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:14:28 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-q2nqj
2026/09/06 18:16:16   18:14:28 INFO Pod/kserve-localmodelnode-agent-mkx4l: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:28 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-mkx4l
2026/09/06 18:16:16   18:14:28 WARN Pod/rhods-dashboard-786954f754-chlsw: Unhealthy - Readiness probe failed: Get "https://10.133.0.129:8444/healthz": dial tcp 10.133.0.129:8444: connect: connection refused
2026/09/06 18:16:16   18:14:28 INFO Pod/kserve-localmodelnode-agent-q2nqj: Killing - Stopping container manager
2026/09/06 18:16:16   18:14:18 WARN Pod/rhods-dashboard-786954f754-chlsw: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:16:16   18:14:18 WARN Pod/rhods-dashboard-786954f754-chlsw: Unhealthy - Readiness probe failed: Get "https://10.133.0.129:8943/healthcheck": dial tcp 10.133.0.129:8943: connect: connection refused
2026/09/06 18:16:16   18:13:59 INFO Pod/rhods-dashboard-786954f754-chlsw: Killing - Stopping container rhods-dashboard
2026/09/06 18:16:16   18:13:59 INFO Pod/rhods-dashboard-786954f754-chlsw: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:16:16   18:13:59 INFO Pod/rhods-dashboard-786954f754-chlsw: Killing - Stopping container core-bff
2026/09/06 18:16:16   18:13:59 INFO ReplicaSet/rhods-dashboard-786954f754: SuccessfulDelete - Deleted pod: rhods-dashboard-786954f754-chlsw
2026/09/06 18:16:16   18:13:56 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-cdjt5_c8478b7a-9d1f-44fc-a8a0-bc784574e28e became leader
2026/09/06 18:16:16   18:13:54 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-c4kpg_5f68c41d-94b4-45ae-8ad2-6e6abb61c5d2 became leader
2026/09/06 18:16:16   18:13:53 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-wg65r_7bd7d4a7-7485-47b4-b430-1613faff9024 became leader
2026/09/06 18:16:16   18:13:49 WARN Pod/rhods-dashboard-786954f754-4njsh: Unhealthy - Readiness probe failed: Get "https://10.132.0.149:8444/healthz": dial tcp 10.132.0.149:8444: connect: connection refused
2026/09/06 18:16:16   18:13:39 WARN Pod/rhods-dashboard-786954f754-4njsh: Unhealthy - Readiness probe failed: Get "https://10.132.0.149:8943/healthcheck": dial tcp 10.132.0.149:8943: connect: connection refused
2026/09/06 18:16:16   18:13:39 WARN Pod/rhods-dashboard-786954f754-4njsh: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/06 18:16:16   18:13:38 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: Started - Started container manager
2026/09/06 18:16:16   18:13:38 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: Created - Created container: manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-q2nqj: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-q2nqj: Created - Created container: manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-q2nqj: Started - Started container manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-q2nqj: AddedInterface - Add eth0 [10.133.0.131/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-q2nqj: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 438ms (438ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-mkx4l: Started - Started container manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-mkx4l: Created - Created container: manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 799ms (799ms including waiting). Image size: 244249302 bytes.
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-mkx4l: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 476ms (476ms including waiting). Image size: 244000959 bytes.
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: Created - Created container: manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: Started - Started container manager
2026/09/06 18:16:16   18:13:38 INFO Pod/odh-model-controller-57457c4c89-wg65r: Started - Started container manager
2026/09/06 18:16:16   18:13:38 INFO Pod/odh-model-controller-57457c4c89-wg65r: Created - Created container: manager
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-mkx4l: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/06 18:16:16   18:13:38 INFO Pod/kserve-localmodelnode-agent-mkx4l: AddedInterface - Add eth0 [10.134.0.96/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:38 INFO Pod/odh-model-controller-57457c4c89-wg65r: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 519ms (519ms including waiting). Image size: 239132625 bytes.
2026/09/06 18:16:16   18:13:38 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 565ms (565ms including waiting). Image size: 235279660 bytes.
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-localmodelnode-agent-q2nqj: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-q2nqj to ip-10-0-134-105.ec2.internal
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-localmodelnode-agent-mkx4l: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-mkx4l to ip-10-0-140-32.ec2.internal
2026/09/06 18:16:16   18:13:37 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/06 18:16:16   18:13:37 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2":
 Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:16:16   18:13:37 INFO Pod/odh-model-controller-57457c4c89-wg65r: AddedInterface - Add eth0 [10.132.0.159/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:37 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: AddedInterface - Add eth0 [10.132.0.157/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:37 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-q2nqj
2026/09/06 18:16:16   18:13:37 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-mkx4l
2026/09/06 18:16:16   18:13:37 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "openshift-ai-inferenceservice-image-volume-scc": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000630000, 1000639999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, p
rovider "kserve-localmodel-permissions-scc": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: Started - Started container manager
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: AddedInterface - Add eth0 [10.132.0.156/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:37 INFO Pod/odh-model-controller-57457c4c89-wg65r: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: AddedInterface - Add eth0 [10.134.0.95/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:37 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/06 18:16:16   18:13:37 INFO Pod/model-serving-api-7df8c9655f-flhvp: Started - Started container server
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: Created - Created container: manager
2026/09/06 18:16:16   18:13:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 422ms (422ms including waiting). Image size: 247667445 bytes.
2026/09/06 18:16:16   18:13:37 INFO Pod/model-serving-api-7df8c9655f-flhvp: Created - Created container: server
2026/09/06 18:16:16   18:13:37 INFO Pod/model-serving-api-7df8c9655f-flhvp: AddedInterface - Add eth0 [10.132.0.158/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:37 INFO Pod/model-serving-api-7df8c9655f-flhvp: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/06 18:16:16   18:13:36 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:13:36 INFO Pod/odh-model-controller-57457c4c89-wg65r: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-wg65r to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:13:36 INFO Pod/model-serving-api-7df8c9655f-flhvp: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-flhvp to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:13:36 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-cdjt5: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-cdjt5 to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:13:36 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-rrhpr: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-rrhpr to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:13:36 INFO Pod/kserve-controller-manager-5fbbd6dd88-c4kpg: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-c4kpg to ip-10-0-140-32.ec2.internal
2026/09/06 18:16:16   18:13:36 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/06 18:16:16   18:13:36 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-c4kpg
2026/09/06 18:16:16   18:13:36 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/06 18:16:16   18:13:36 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-flhvp
2026/09/06 18:16:16   18:13:36 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/06 18:16:16   18:13:36 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/06 18:16:16   18:13:36 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-wg65r
2026/09/06 18:16:16   18:13:36 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/06 18:16:16   18:13:36 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-cdjt5
2026/09/06 18:16:16   18:13:36 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-rrhpr
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Created - Created container: core-bff
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Started - Started container core-bff
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Created - Created container: rhods-dashboard
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Started - Started container rhods-dashboard
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Created - Created container: kube-rbac-proxy
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Started - Started container kube-rbac-proxy
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:16:16   18:13:21 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 447ms (447ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:16:16   18:13:20 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: AddedInterface - Add eth0 [10.133.0.130/23] from ovn-kubernetes
2026/09/06 18:16:16   18:13:20 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:16:16   18:13:20 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 292ms (292ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:16:16   18:13:19 INFO Pod/rhods-dashboard-86c5fb98b4-nkl5c: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-86c5fb98b4-nkl5c to ip-10-0-134-105.ec2.internal
2026/09/06 18:16:16   18:13:19 INFO Pod/rhods-dashboard-786954f754-4njsh: Killing - Stopping container kube-rbac-proxy
2026/09/06 18:16:16   18:13:19 INFO Deployment/rhods-dashboard: ScalingReplicaSet - (combined from similar events): Scaled down replica set rhods-dashboard-786954f754 from 2 to 1
2026/09/06 18:16:16   18:13:19 INFO ReplicaSet/rhods-dashboard-786954f754: SuccessfulDelete - Deleted pod: rhods-dashboard-786954f754-4njsh
2026/09/06 18:16:16   18:13:19 INFO Pod/rhods-dashboard-786954f754-4njsh: Killing - Stopping container core-bff
2026/09/06 18:16:16   18:13:19 INFO ReplicaSet/rhods-dashboard-86c5fb98b4: SuccessfulCreate - Created pod: rhods-dashboard-86c5fb98b4-nkl5c
2026/09/06 18:16:16   18:13:19 INFO Pod/rhods-dashboard-786954f754-4njsh: Killing - Stopping container rhods-dashboard
2026/09/06 18:16:16   18:13:12 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-rc68t_39971a14-9d3d-4f85-b17e-a0830baead6d became leader
2026/09/06 18:16:16   18:13:08 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-sv2wl_a63b0a27-a2c4-4f59-bc2a-1dc9a8abec9a became leader
2026/09/06 18:16:16   18:12:54 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-rc68t: Started - Started container manager
2026/09/06 18:16:16   18:12:54 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-rc68t: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" in 1.266s (1.266s including waiting). Image size: 158342259 bytes.
2026/09/06 18:16:16   18:12:54 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-rc68t: Created - Created container: manager
2026/09/06 18:16:16   18:12:53 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-rc68t: Pulling - Pulling image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35"
2026/09/06 18:16:16   18:12:53 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-rc68t: AddedInterface - Add eth0 [10.132.0.155/23] from ovn-kubernetes
2026/09/06 18:16:16   18:12:52 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-rc68t: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-rc68t to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:12:52 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/06 18:16:16   18:12:52 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-rc68t
2026/09/06 18:16:16   18:12:48 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-g4l4h: Created - Created container: manager
2026/09/06 18:16:16   18:12:48 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-g4l4h: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" in 2.429s (2.429s including waiting). Image size: 198393268 bytes.
2026/09/06 18:16:16   18:12:48 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" in 2.446s (2.446s including waiting). Image size: 227888997 bytes.
2026/09/06 18:16:16   18:12:48 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: Started - Started container manager
2026/09/06 18:16:16   18:12:48 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: Created - Created container: manager
2026/09/06 18:16:16   18:12:48 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-g4l4h: Started - Started container manager
2026/09/06 18:16:16   18:12:46 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-g4l4h: AddedInterface - Add eth0 [10.132.0.154/23] from ovn-kubernetes
2026/09/06 18:16:16   18:12:46 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-g4l4h: Pulling - Pulling image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2"
2026/09/06 18:16:16   18:12:46 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: Pulling - Pulling image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b"
2026/09/06 18:16:16   18:12:46 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: AddedInterface - Add eth0 [10.132.0.153/23] from ovn-kubernetes
2026/09/06 18:16:16   18:12:45 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-g4l4h: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-g4l4h to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:12:45 INFO Pod/kserve-module-controller-manager-66ff7c977-sv2wl: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-sv2wl to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:12:45 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/06 18:16:16   18:12:45 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/06 18:16:16   18:12:45 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-sv2wl
2026/09/06 18:16:16   18:12:45 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-g4l4h
2026/09/06 18:16:16   18:12:42 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Created - Created container: core-bff
2026/09/06 18:16:16   18:12:42 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 418ms (418ms including waiting). Image size: 181738206 bytes.
2026/09/06 18:16:16   18:12:42 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Started - Started container core-bff
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Started - Started container rhods-dashboard
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Created - Created container: kube-rbac-proxy
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Started - Started container kube-rbac-proxy
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Created - Created container: rhods-dashboard
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 430ms (430ms including waiting). Image size: 812543307 bytes.
2026/09/06 18:16:16   18:12:41 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/06 18:16:16   18:12:40 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-86c5fb98b4-2g7vz to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:12:40 INFO Pod/eval-hub-ui-869b89d9b4-nfwmt: Killing - Stopping container eval-hub-ui
2026/09/06 18:16:16   18:12:40 INFO Pod/rhods-dashboard-86c5fb98b4-2g7vz: AddedInterface - Add eth0 [10.132.0.152/23] from ovn-kubernetes
2026/09/06 18:16:16   18:12:40 INFO ReplicaSet/rhods-dashboard-86c5fb98b4: SuccessfulCreate - Created pod: rhods-dashboard-86c5fb98b4-2g7vz
2026/09/06 18:16:16   18:12:33 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-7nf7q: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:25 INFO Pod/kserve-module-controller-manager-66ff7c977-85j4c: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:25 INFO Lease/b7e9931f.trustyai.opendatahub.io: LeaderElection - trustyai-service-operator-controller-manager-d6bf47757-7nf7q_1a7104d9-035c-4ca1-9e64-60689f3c01fd became leader
2026/09/06 18:16:16   18:12:23 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:12:23 INFO Pod/model-serving-api-7df8c9655f-j9bbx: Killing - Stopping container server
2026/09/06 18:16:16   18:12:23 INFO Pod/odh-model-controller-57457c4c89-t5x6j: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:23 INFO Pod/kserve-controller-manager-5fbbd6dd88-8g28k: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:23 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-cr6wd: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:23 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-djc6x: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:22 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/06 18:16:16   18:12:22 INFO Pod/kserve-localmodelnode-agent-nmql4: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:22 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-927tf
2026/09/06 18:16:16   18:12:22 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-nmql4
2026/09/06 18:16:16   18:12:22 INFO Pod/kserve-localmodelnode-agent-927tf: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:08 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-7nf7q: AddedInterface - Add eth0 [10.132.0.151/23] from ovn-kubernetes
2026/09/06 18:16:16   18:12:08 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-7nf7q: Pulled - Container image "quay.io/rhoai/odh-trustyai-service-operator-rhel9@sha256:e41a5793f10820747e807d9165c73298387d7feefc81f4b0d9b70e911d6dbe40" already present on machine
2026/09/06 18:16:16   18:12:08 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-7nf7q: Created - Created container: manager
2026/09/06 18:16:16   18:12:08 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-7nf7q: Started - Started container manager
2026/09/06 18:16:16   18:12:07 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-7nf7q: Scheduled - Successfully assigned redhat-ods-applications/trustyai-service-operator-controller-manager-d6bf47757-7nf7q to ip-10-0-141-252.ec2.internal
2026/09/06 18:16:16   18:12:07 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-lltjx: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:07 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-w67vf: Killing - Stopping container manager
2026/09/06 18:16:16   18:12:07 INFO ReplicaSet/trustyai-service-operator-controller-manager-d6bf47757: SuccessfulCreate - Created pod: trustyai-service-operator-controller-manager-d6bf47757-7nf7q
2026/09/06 18:16:16   18:12:07 INFO Deployment/trustyai-service-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trustyai-service-operator-controller-manager-d6bf47757 from 0 to 1
2026/09/06 18:16:16 === RESOURCE QUOTAS ===
2026/09/06 18:16:16 No resource quotas found in monitored namespaces
2026/09/06 18:16:16 Classification for components: infrastructure/storage (code=1006, confidence=medium)
2026/09/06 18:16:16   Evidence: event Pod/kserve-localmodelnode-agent-rw779: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"storage","error_code":1006,"evidence":["event Pod/kserve-localmodelnode-agent-rw779: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found"],"confidence":"medium"}
2026/09/06 18:16:16 Diagnostics complete for failed test: components
    controller_test.go:490: Stopping: components test failed.
2026/09/06 18:16:16 Circuit breaker: failure classified as infrastructure/storage (code=1006)
2026/09/06 18:16:16 Circuit breaker: infrastructure/unknown failure recorded (2/3)
    controller_test.go:762: Skipping tests for services/monitoring
Deletion Policy: Never. Skipping deletion tests.
2026/09/06 18:16:17 Circuit breaker: no trips during this run (threshold=3)
    controller_test.go:762: Skipping tests for services/monitoring]]></system-out>
	</testsuite>
</testsuites>
