<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="246" failures="5" skipped="61">
	<testsuite name="" tests="246" failures="5" errors="0" id="0" hostname="e2e-cli-nmanos-ephc-rhoai-smoke-xw7dw-test-platform-pod" skipped="61" time="10831.680" timestamp="2026-09-07T22:02:31Z">
		<testcase name="TestOdhOperator" classname="" time="1740.780">
			<system-out><![CDATA[2026/09/07 21:33:31 ARTIFACT_DIR not set, skipping periodic metrics collection
2026-09-07T21:33:31Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests" classname="" time="0.200">
			<system-out><![CDATA[2026-09-07T21:33:31Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Datascience_Cluster_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataScienceCluster_Initialization_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_FeatureTracker_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Ray_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_TrustyAI_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Kueue_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_DataSciencePipelines_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Auth_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/Operator_Manager_E2E_Tests/Validate_CRDs_owned_by_the_operator/Validate_Platform_CRD" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests" classname="" time="0.480">
			<system-out><![CDATA[2026-09-07T21:33:31Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_HardwareProfile_resource" classname="" time="10.150"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_owned_namespaces_exist" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_default_NetworkPolicy_exist" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_denies_invalid_PEM_on_update" classname="" time="0.090"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_valid_PEM_on_update" classname="" time="0.140"></testcase>
		<testcase name="TestOdhOperator/DSCInitialization_and_DataScienceCluster_validation_E2E_Tests/Validate_DSCI_webhook_allows_invalid_PEM_when_Removed" classname="" time="0.110"></testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests" classname="" time="11.150">
			<system-out><![CDATA[2026-09-07T21:33:42Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Ensuring clean slate: setting all components to Removed
    controller_test.go:762: Patched Subscription to remove env vars: [RHAI_VERSION CI]]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_in-tree_gates_block_and_unblock_provisioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_admin_ack_gates_block_and_unblock_provisioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_partial_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_runlevel_gating_and_convergence" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_PlatformReady_condition_on_component_CRs" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_component_stability_across_enable/disable_cycles_(RHOAIENG-73142)" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/DAG_Ordering_E2E_Tests/Validate_DAG_cleanup" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2 Tier3]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/monitoring" classname="" time="0.140">
			<skipped message="Skipped"><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
    controller_test.go:762: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components" classname="" time="1718.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway" classname="" time="70.710"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard" classname="" time="177.370">
			<system-out><![CDATA[2026-09-07T21:34:34Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines" classname="" time="86.580">
			<system-out><![CDATA[2026-09-07T21:34:35Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator" classname="" time="65.760"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve" classname="" time="424.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator" classname="" time="40.870"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx" classname="" time="61.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray" classname="" time="91.440"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator" classname="" time="121.200">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer" classname="" time="40.930"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches" classname="" time="41.650"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_enabled" classname="" time="60.370">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_enabled" classname="" time="101.370">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_enabled" classname="" time="30.900">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_enabled" classname="" time="30.410">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_enabled" classname="" time="50.660">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}
2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_enabled" classname="" time="110.660"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_enabled" classname="" time="30.490">
			<system-out><![CDATA[2026-09-07T21:33:53Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade_from_in-tree:_selector_migration" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_component_enabled" classname="" time="0.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_operator_deployed" classname="" time="25.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_created" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_CR_ready" classname="" time="30.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/mcplifecycleoperator/Validate_component_disabled" classname="" time="10.200"></testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_module_handler_projects_DSC_config_to_Module_CR" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/trainer/Validate_component_disabled" classname="" time="10.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_operator_deployment" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_workbenches_namespace_configuration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_module_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_ImageStreams_available" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_MLflow_integration" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier2]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_WorkbenchesV2_default_Removed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery" classname="" time="0.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No ConfigMap resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.020">
			<system-out><![CDATA[    helper_test.go:156: No Service resources found for component Workbenches, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Deployment resources found for component Workbenches, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_component_disabled" classname="" time="10.400"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_enabled" classname="" time="30.520">
			<system-out><![CDATA[2026-09-07T21:34:34Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_component_enabled" classname="" time="40.650"></testcase>
		<testcase name="TestOdhOperator/components/group_1/workbenches/Validate_WorkbenchesV2_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_enabled" classname="" time="20.420"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ogx/Validate_component_disabled" classname="" time="10.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_feast-operator_deployed_by_module" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_upgrade:_existing_operands_preserved" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/feastoperator/Validate_module_disabled_cleanup" classname="" time="10.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_with_mixed_enablement" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_submodule_conditions_when_parent_disabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/aigateway/Validate_component_disabled" classname="" time="10.210"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_conditions" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_operands_have_OwnerReferences" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources" classname="" time="20.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_update_operand_resources/deployment_data-science-pipelines-operator-controller-manager" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_operands_have_OwnerReferences" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources" classname="" time="20.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_update_operand_resources/deployment_kuberay-operator" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_operands_have_OwnerReferences" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_update_operand_resources" classname="" time="20.050"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_argoWorkflowsControllers_options" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery" classname="" time="35.390"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="15.190"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-config" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_data-science-pipelines-operator-dspo-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_releases" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_platform_release" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery" classname="" time="30.380"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_ray-config-bf2bfgg6b4" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_workflow-controller-configmap" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="10.190"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-operator" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_data-science-pipelines-operator-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_spec" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_operands_have_OwnerReferences" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_no_Kserve_FeatureTrackers" classname="" time="0.010"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_VAP_created_when_kserve_is_enabled" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources" classname="" time="100.270"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_dynamically_watches_operands" classname="" time="10.170"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kuberay-webhook-service" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.120"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_data-science-pipelines-operator-controller-manager" classname="" time="15.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="15.110"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kuberay-operator" classname="" time="15.090"></testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_operator_bootstrap_resources" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_env_var_injection" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_SparkPi_workload_execution" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_ScheduledSparkApplication_workload_execution" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_CR_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_module_operator_Deployment_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/sparkoperator/Validate_component_disabled" classname="" time="10.240"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_CRDs_reinstated" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_VAP_blocks_dashboard_HardwareProfile_and_AcceleratorProfile_creation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery" classname="" time="106.260"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="105.730"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_agent-ops-params" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_automl-params" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/datasciencepipelines/Validate_component_disabled" classname="" time="10.400"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_kserve-localmodel-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/ray/Validate_component_disabled" classname="" time="10.220"></testcase>
		<testcase name="TestOdhOperator/components/group_1/dashboard/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_autorag-params" classname="" time="95.080">
			<failure message="Failed"></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_llmisvc-controller-manager" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_model-serving-api" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_update_operand_resources/deployment_odh-model-controller" classname="" time="20.040"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_releases" classname="" time="0.030"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_well-known_LLMInferenceServiceConfig_versioning" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_subscription_dependency_conditions" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_platform_config_ConfigMap" classname="" time="0.030">
			<system-out><![CDATA[    helper_test.go:156: Verifying platform config ConfigMap odh-kserve-config exists in namespace redhat-ods-applications.
    helper_test.go:156: Non-XKS platform: verifying cert-manager CA keys are absent.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_ModelRegistry_state_propagation" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery" classname="" time="211.640"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="20.290"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_inferenceservice-config" classname="" time="5.080"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-modelcache-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_kserve-parameters" classname="" time="5.060"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery/ConfigMap_odh-model-controller-parameters" classname="" time="5.060">
			<system-out><![CDATA[    test_context_test.go:1326: 
        Timed out after 90.000s.
        Resource ConfigMap redhat-ods-applications/autorag-params was not properly recreated with new identity
        The function passed to Eventually failed at /workspace/tests/e2e/test_context_test.go:1307 with:
        Expected ConfigMap redhat-ods-applications/autorag-params to be recreated
        Expected
            <*unstructured.Unstructured | 0x0>: nil
        not to be nil
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="45.650"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-controller-manager-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_kserve-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-controller-manager-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_llmisvc-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_localmodel-webhook-server-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_model-serving-api" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-metrics-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Service_deletion_recovery/Service_odh-model-controller-webhook-service" classname="" time="5.070"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="145.700"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-controller-manager" classname="" time="35.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_kserve-localmodel-controller-manager" classname="" time="35.140"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_llmisvc-controller-manager" classname="" time="35.150"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_model-serving-api" classname="" time="15.100"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_resource_deletion_recovery/Deployment_deletion_recovery/deployment_odh-model-controller" classname="" time="25.130"></testcase>
		<testcase name="TestOdhOperator/components/group_1/kserve/Validate_component_disabled" classname="" time="10.370"></testcase>
		<testcase name="TestOdhOperator/components/group_2" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue" classname="" time="93.130"></testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry" classname="" time="70.640">
			<system-out><![CDATA[2026-09-07T21:40:57Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_enabled" classname="" time="50.260">
			<system-out><![CDATA[2026-09-07T21:40:57Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_error_with_ocp_kueue-operator_not_installed" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_removed_to_unmanaged_transition" classname="" time="10.800">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Creating test namespace (test-kueue-managed-dafiuq9ve3cs7m7v0qfg) with Kueue management annotation.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:156: Updating the management state of the component in the DataScienceCluster to Unmanaged with autoCreateQueues=true (expecting NotReady initially due to missing operator).
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Verifying the component in the DataScienceCluster transitions to Ready state.
    helper_test.go:156: Verifying Kueue configuration (cluster) is created with all expected frameworks.
    helper_test.go:156: Verifying default ClusterQueue and LocalQueue resources are created in namespace test-kueue-managed-dafiuq9ve3cs7m7v0qfg.
    helper_test.go:156: Verifying default resource flavor (default-flavor) is created.
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind']]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_platform_release" classname="" time="0.020"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_unmanaged_to_removed_transition" classname="" time="51.150">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Defining expected conditions for Unmanaged state (Ready=True).
    helper_test.go:156: Updating DSC default-dsc to set Kueue managementState to Unmanaged with autoCreateQueues=true.
    helper_test.go:156: Creating test namespace (test-kueue-managed-dafiut1ve3cs7m7v0qg0) with Kueue management annotation.
    helper_test.go:156: Verifying default Kueue resources exist in namespace test-kueue-managed-dafiut1ve3cs7m7v0qg0.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_module_CR_spec" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_registries_namespace_created" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_releases_mirrored_to_DSC" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipping test: passed tag: Smoke, test tags: [Tier1]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/modelregistry/Validate_component_disabled" classname="" time="20.230">
			<system-out><![CDATA[    helper_test.go:156: Verifying default resource flavor (default-flavor) is created.
    helper_test.go:156: Updating DSC default-dsc to set Kueue managementState to Removed.
    helper_test.go:156: Verifying default resources are still present in namespace test-kueue-managed-dafiut1ve3cs7m7v0qg0.
    helper_test.go:156: Cleaning up Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_autoCreateQueues_disabled_skips_queue_creation" classname="" time="20.750">
			<system-out><![CDATA[    helper_test.go:156: Setting Kueue component (default-kueue) to Removed mode to start with a clean state.
    helper_test.go:156: Cleaning up any existing Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    helper_test.go:156: Ensuring LWS Operator is installed (namespace=openshift-lws-operator, name=leader-worker-set).
    helper_test.go:156: Ensuring LeaderWorkerSetOperator CR exists (namespace=openshift-lws-operator, name=cluster).]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery" classname="" time="0.040"></testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ConfigMap_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No ConfigMap resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Service_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Service resources found for component Kueue, skipping]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/RBAC_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to external dependency timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/ServiceAccount_deletion_recovery" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    helper_test.go:156: Skipped due to token refresh timing issues]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_resource_deletion_recovery/Deployment_deletion_recovery" classname="" time="0.010">
			<system-out><![CDATA[    helper_test.go:156: No Deployment resources found for component Kueue, skipping deletion recovery test]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_2/kueue/Validate_component_disabled" classname="" time="10.210">
			<system-out><![CDATA[    helper_test.go:156: Setting DSC component default-kueue to Removed state.
    helper_test.go:156: Verifying no Deployments remain in namespace redhat-ods-applications with part-of=kueue label.
    helper_test.go:156: Ensuring the component CR /default-kueue is removed.]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator" classname="" time="338.120"></testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai" classname="" time="600.160">
			<system-out><![CDATA[2026-09-07T21:42:31Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></system-out>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/mlflowoperator/Validate_component_enabled" classname="" time="337.960">
			<failure message="Failed"><![CDATA[2026-09-07T21:42:31Z	INFO	Cluster config	{"Operator Namespace": "redhat-ods-operator", "Application Namespace": "redhat-ods-applications", "Release": {"name":"OpenShift AI Self-Managed","version":"3.6.0-ea.1"}, "Cluster": {"type":"OpenShift","version":"4.21.30"}}]]></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_3/trustyai/Validate_component_enabled" classname="" time="600.000">
			<failure message="Failed"><![CDATA[    resource_applier_test.go:211: 
        Timed out after 337.884s.
        The function passed to Eventually failed at /workspace/tests/e2e/resource_applier_test.go:287 with:
        Error occurred while applying the resource 'default-dsc' of kind 'DataScienceCluster': Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host - error from a previous attempt: http2: client connection lost
        Unexpected error:
            <*rest.wrapPreviousError | 0x34e0257a8840>: 
            Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host - error from a previous attempt: http2: client connection lost
            {
                currentErr: <*url.Error | 0x34e0268eb8f0>{
                    Op: "Get",
                    URL: "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc",
                    Err: <*net.OpError | 0x34e02563b220>{
                        Op: "dial",
                        Net: "tcp",
                        Source: nil,
                        Addr: nil,
                        Err: <*net.DNSError | 0x34e02563b1d0>{
                            UnwrapErr: nil,
                            Err: "no such host",
                            Name: "a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com",
                            Server: "172.30.0.10:53",
                            IsTimeout: false,
                            IsTemporary: false,
                            IsNotFound: true,
                        },
                    },
                },
                previousError: <*errors.errorString | 0x34e026241c40>{
                    s: "http2: client connection lost",
                },
            }
        occurred
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4" classname="" time="0.000"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve" classname="" time="600.020"></testcase>
		<testcase name="TestOdhOperator/components/group_4/kueue" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:762: Skipping: flaky due to namespace stuck in Terminating state. See RHOAIENG-46773 and RHOAIENG-46774.]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer" classname="" time="300.020"></testcase>
		<testcase name="TestOdhOperator/components/group_4/trainer/Validate_component_enabled" classname="" time="300.010">
			<failure message="Failed"><![CDATA[    test_context_test.go:120: Warning: cluster.Init failed: failed to get server groups: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_4/kserve/Validate_component_enabled" classname="" time="600.010">
			<failure message="Failed"><![CDATA[    resource_applier_test.go:211: 
        Timed out after 300.009s.
        The function passed to Eventually failed at /workspace/tests/e2e/resource_applier_test.go:287 with:
        Error occurred while applying the resource 'default-dsc' of kind 'DataScienceCluster': failed to get server groups: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
        Unexpected error:
            <*fmt.wrapError | 0x34e0260beae0>: 
            failed to get server groups: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
            {
                msg: "failed to get server groups: Get \"https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api\": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host",
                err: <*url.Error | 0x34e026a59050>{
                    Op: "Get",
                    URL: "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api",
                    Err: <*net.OpError | 0x34e0262f4af0>{
                        Op: "dial",
                        Net: "tcp",
                        Source: nil,
                        Addr: nil,
                        Err: <*net.DNSError | 0x34e026258640>{
                            UnwrapErr: nil,
                            Err: "no such host",
                            Name: "a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com",
                            Server: "172.30.0.10:53",
                            IsTimeout: false,
                            IsTemporary: false,
                            IsNotFound: true,
                        },
                    },
                },
            }
        occurred
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test]]></failure>
		</testcase>
		<testcase name="TestOdhOperator/components/group_5" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:762: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/services" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:493: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Operator_Resilience_E2E_Tests" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:497: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/V2_to_V3_upgrade_E2E_Tests" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:502: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/Deletion_ConfigMap_E2E_Tests" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:508: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestOdhOperator/upgrade_DSC_and_DSCI_v1_API" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    controller_test.go:513: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestLogPodsSectionContinuesWithPartialData" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesComponentsMatchRealTestGroup" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesServicesMatchRealTestGroup" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/space-separated_matches_the_real_contract" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/comma-separated_collapses_into_one_malformed_name_--_this_is_the_bug,_not_the_fix" classname="" time="0.000"></testcase>
		<testcase name="TestScopeRulesEnvVarFormatIsSpaceSeparated/single_name_has_no_separator_to_get_wrong" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/all_enabled_when_no_flags" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/explicit_inclusion_limits_to_named_tests" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/exclusion_removes_from_full_set" classname="" time="0.000"></testcase>
		<testcase name="TestResolveEnabledTests/inclusion_and_exclusion_combined" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/runs_only_the_named_test/monitoring" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:78: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/skips_when_test_not_found/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:85: Test nonexistent is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunSingle/respects_flags_disabling_the_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:94: Test monitoring is disabled by flags in group services]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/runs_all_except_excluded/services/group_1" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:116: CIRCUIT BREAKER OPEN: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_when_group_is_disabled/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:128: Test group services is disabled]]></skipped>
		</testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test" classname="" time="0.000"></testcase>
		<testcase name="TestRunExcluding/skips_entirely_when_excluding_only_remaining_test/inner" classname="" time="0.000">
			<skipped message="Skipped"><![CDATA[    testgroup_test.go:142: No tests remaining in group single after excluding only]]></skipped>
		</testcase>
		<testcase name="TestIsTestEnabled" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/enabled_by_default" classname="" time="0.000"></testcase>
		<testcase name="TestIsTestEnabled/disabled_by_exclusion_flag" classname="" time="0.000"></testcase>
		<system-out><![CDATA[    controller_test.go:762: Restoring operator to original version for Phase 3
    controller_test.go:762: Patched Subscription to remove env vars: [RHAI_VERSION CI]
    controller_test.go:762: Waiting for DSC to fully reconcile at original version
    dag_ordering_test.go:193: Cleanup: setting all components to Removed, deleting quota, and restoring operator env vars
    dag_ordering_test.go:196: Patched Subscription to remove env vars: [RHAI_VERSION CI]
2026/09/07 21:37:30 TEST FAILURE DETECTED: ConfigMap deletion recovery
2026/09/07 21:37:30 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:ConfigMap deletion recovery) ===
2026/09/07 21:37:31 === CLUSTER STATE ===
2026/09/07 21:37:31 Node: ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:37:31   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:37:31   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:37:31   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:37:31   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:37:31 Node: ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:37:31   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:37:31   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:37:31   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:37:31   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:37:31 Node: ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:37:31   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:37:31   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:37:31   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:37:31   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:37:31 === DEPLOYMENTS ===
2026/09/07 21:37:31 Namespace: redhat-ods-applications
2026/09/07 21:37:31   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 21:37:31   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 21:37:31   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment kserve-controller-manager: 2/2 (READY)
2026/09/07 21:37:31   Deployment kserve-localmodel-controller-manager: 2/2 (READY)
2026/09/07 21:37:31   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/07 21:37:31   Deployment llmisvc-controller-manager: 2/2 (READY)
2026/09/07 21:37:31   Deployment maas-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment model-serving-api: 2/2 (READY)
2026/09/07 21:37:31   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment odh-model-controller: 2/2 (READY)
2026/09/07 21:37:31   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 21:37:31 Namespace: kube-system
2026/09/07 21:37:31 === PODS ===
2026/09/07 21:37:31 Failed to collect pod data: redhat-ods-applications/rhods-dashboard-858fdb6f4d-wqtjf: container kube-rbac-proxy not ready
2026/09/07 21:37:31 Continuing with partial pod data to collect logs from problematic containers
2026/09/07 21:37:31 Namespace: redhat-ods-applications
2026/09/07 21:37:31   Pod rhods-dashboard-858fdb6f4d-wqtjf: Running
2026/09/07 21:37:31     Container kube-rbac-proxy: ready=false
2026/09/07 21:37:31 Namespace kube-system: no problematic pods found
2026/09/07 21:37:31 === OPERATOR STATUS ===
2026/09/07 21:37:31 Operator deployment: 3/3 ready
2026/09/07 21:37:31 === DSCI STATUS ===
2026/09/07 21:37:31 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 21:37:31 === DSC STATUS ===
2026/09/07 21:37:31 DSC default-dsc:
2026/09/07 21:37:31   Ready: True - 
2026/09/07 21:37:31   ProvisioningSucceeded: True - 
2026/09/07 21:37:31   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 21:37:31   ModulesReady: True - 
2026/09/07 21:37:31   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 21:37:31   DashboardReady: True - 
2026/09/07 21:37:31   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   KserveLLMInferenceServiceDependencies: True - 
2026/09/07 21:37:31   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/07 21:37:31   KserveReady: True - 
2026/09/07 21:37:31   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 21:37:31   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   RayReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   TrainerReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 21:37:31 === RECENT EVENTS ===
2026/09/07 21:37:31   21:37:29 WARN Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get "https://10.133.0.117:8444/healthz": dial tcp 10.133.0.117:8444: connect: connection refused
2026/09/07 21:37:31   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:37:17 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulDelete - Deleted pod: rhods-dashboard-858fdb6f4d-wqtjf
2026/09/07 21:37:31   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:37:17 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-858fdb6f4d from 1 to 0
2026/09/07 21:37:31   21:37:05 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe failed: Get "https://10.132.0.97:8444/healthz": dial tcp 10.132.0.97:8444: connect: connection refused
2026/09/07 21:37:31   21:36:58 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Started - Started container manager
2026/09/07 21:37:31   21:36:57 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Created - Created container: manager
2026/09/07 21:37:31   21:36:57 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 1.904s (1.904s including waiting). Image size: 239132625 bytes.
2026/09/07 21:37:31   21:36:56 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 21:37:31   21:36:55 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-xg8fl to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:36:55 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/07 21:37:31   21:36:55 INFO Pod/odh-model-controller-57457c4c89-xg8fl: AddedInterface - Add eth0 [10.134.0.112/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:55 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-xg8fl
2026/09/07 21:37:31   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 354ms (354ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: core-bff
2026/09/07 21:37:31   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container core-bff
2026/09/07 21:37:31   21:36:40 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:36:40 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe failed: Get "https://10.132.0.97:8943/healthcheck": dial tcp 10.132.0.97:8943: connect: connection refused
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 416ms (416ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: AddedInterface - Add eth0 [10.132.0.102/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-764dcdbfcc-4chfj to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:36:38 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-558d97b5db from 1 to 0
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:36:38 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-764dcdbfcc from 1 to 2
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:36:38 INFO ReplicaSet/rhods-dashboard-764dcdbfcc: SuccessfulCreate - Created pod: rhods-dashboard-764dcdbfcc-4chfj
2026/09/07 21:37:31   21:36:38 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulDelete - Deleted pod: rhods-dashboard-558d97b5db-nn7q2
2026/09/07 21:37:31   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.42s (1.42s including waiting). Image size: 172459788 bytes.
2026/09/07 21:37:31   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Started - Started container server
2026/09/07 21:37:31   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Created - Created container: server
2026/09/07 21:37:31   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-xfmcz to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:36:35 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/07 21:37:31   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: AddedInterface - Add eth0 [10.134.0.111/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 21:37:31   21:36:35 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-xfmcz
2026/09/07 21:37:31   21:36:30 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:36:30 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8943/healthcheck": dial tcp 10.132.0.101:8943: connect: connection refused
2026/09/07 21:37:31   21:36:27 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
2026/09/07 21:37:31   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 329ms (329ms including waiting). Image size: 235279660 bytes.
2026/09/07 21:37:31   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Started - Started container manager
2026/09/07 21:37:31   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Created - Created container: manager
2026/09/07 21:37:31   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-j2nvj to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:36:15 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/07 21:37:31   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: AddedInterface - Add eth0 [10.134.0.110/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 21:37:31   21:36:15 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-j2nvj
2026/09/07 21:37:31   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe failed: Get "https://10.133.0.115:8943/healthcheck": dial tcp 10.133.0.115:8943: connect: connection refused
2026/09/07 21:37:31   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe failed: Get "https://10.133.0.115:8444/healthz": dial tcp 10.133.0.115:8444: connect: connection refused
2026/09/07 21:37:31   21:36:05 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:05 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:04 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 400ms (400ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 324ms (324ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: core-bff
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container core-bff
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-764dcdbfcc-lt7q7 to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: AddedInterface - Add eth0 [10.133.0.119/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:36:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-764dcdbfcc from 0 to 1
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:36:00 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulDelete - Deleted pod: rhods-dashboard-858fdb6f4d-22p4c
2026/09/07 21:37:31   21:36:00 INFO ReplicaSet/rhods-dashboard-764dcdbfcc: SuccessfulCreate - Created pod: rhods-dashboard-764dcdbfcc-lt7q7
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:36:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-858fdb6f4d from 2 to 1
2026/09/07 21:37:31   21:35:59 INFO Pod/automl-ui-74d9b97d49-stzqx: Killing - Stopping container automl-ui
2026/09/07 21:37:31   21:35:59 INFO Pod/autorag-ui-58f548cbd-8n44h: Killing - Stopping container autorag-ui
2026/09/07 21:37:31   21:35:59 WARN Endpoints/odh-dashboard-autorag-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-autorag-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-autorag-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-autorag-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: c737a2c7-49fc-4ca4-a807-a07ef76ed351, UID in object meta: 
2026/09/07 21:37:31   21:35:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 349ms (349ms including waiting). Image size: 244249302 bytes.
2026/09/07 21:37:31   21:35:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Created - Created container: manager
2026/09/07 21:37:31   21:35:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Started - Started container manager
2026/09/07 21:37:31   21:35:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-lg2tv to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:55 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:35:55 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-lg2tv
2026/09/07 21:37:31   21:35:55 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 1 to 2
2026/09/07 21:37:31   21:35:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 21:37:31   21:35:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: AddedInterface - Add eth0 [10.133.0.118/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:51 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:51 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:46 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:46 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Started - Started container core-bff
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 362ms (362ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 347ms (347ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:44 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:44 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-858fdb6f4d-22p4c to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:44 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-558d97b5db from 2 to 1
2026/09/07 21:37:31   21:35:44 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulCreate - Created pod: rhods-dashboard-858fdb6f4d-22p4c
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:35:44 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Killing - Stopping container webhook
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: AddedInterface - Add eth0 [10.132.0.101/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:44 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Killing - Stopping container controller
2026/09/07 21:37:31   21:35:44 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-858fdb6f4d from 1 to 2
2026/09/07 21:37:31   21:35:44 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulDelete - Deleted pod: rhods-dashboard-558d97b5db-5v54m
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Created - Created container: kuberay-operator
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Started - Started container kuberay-operator
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: AddedInterface - Add eth0 [10.134.0.109/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:40 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-4k8xx to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:40 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:35:40 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-4k8xx
2026/09/07 21:37:31   21:35:40 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 21:37:31   21:35:40 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:35:38 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.246s (1.246s including waiting). Image size: 178395693 bytes.
2026/09/07 21:37:31   21:35:38 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Started - Started container manager
2026/09/07 21:37:31   21:35:38 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Created - Created container: manager
2026/09/07 21:37:31   21:35:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Created - Created container: manager
2026/09/07 21:37:31   21:35:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 1.871s (1.871s including waiting). Image size: 247667445 bytes.
2026/09/07 21:37:31   21:35:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Started - Started container manager
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f8997dhh to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:36 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f8997dhh
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: AddedInterface - Add eth0 [10.134.0.108/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:36 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 21:37:31   21:35:35 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-n8xx4 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:35 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-n8xx4
2026/09/07 21:37:31   21:35:35 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: AddedInterface - Add eth0 [10.132.0.100/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:35 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 21:37:31   21:35:35 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 1 to 2
2026/09/07 21:37:31   21:35:17 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Started - Started container manager
2026/09/07 21:37:31   21:35:17 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" in 1.42s (1.42s including waiting). Image size: 170934264 bytes.
2026/09/07 21:37:31   21:35:17 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Created - Created container: manager
2026/09/07 21:37:31   21:35:16 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-nmnhd_91fbcd1f-6e9c-4b7e-a296-8a92e6a4eed0 became leader
2026/09/07 21:37:31   21:35:15 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-6qzxp to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:15 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 1 to 2
2026/09/07 21:37:31   21:35:15 INFO Pod/dashboard-operator-669d6564bd-6qzxp: AddedInterface - Add eth0 [10.132.0.99/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:15 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520"
2026/09/07 21:37:31   21:35:15 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-ph2cn_856dd34e-3152-48ff-9d1c-a5c8dba42773 became leader
2026/09/07 21:37:31   21:35:15 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-6qzxp
2026/09/07 21:37:31   21:35:14 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-k7j5f_d06a40c7-50a7-4800-9783-43c5f10727a0 became leader
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Created - Created container: manager
2026/09/07 21:37:31   21:35:09 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3" in 3.054s (3.055s including waiting). Image size: 184646446 bytes.
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Started - Started container manager
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 385ms (385ms including waiting). Image size: 244000959 bytes.
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 323ms (323ms including waiting). Image size: 244000959 bytes.
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Created - Created container: manager
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Started - Started container manager
2026/09/07 21:37:31   21:35:09 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Created - Created container: agent-ops-ui
2026/09/07 21:37:31   21:35:09 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Started - Started container agent-ops-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-95jw9: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-95jw9 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-8dkgk: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-8dkgk to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:08 INFO Pod/autorag-ui-58f548cbd-8n44h: Created - Created container: autorag-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/maas-ui-68f944f6b6-jh699: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe" in 4.366s (4.366s including waiting). Image size: 189581000 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/notebooks-ui-6577cc467-27k2h: Created - Created container: notebooks-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/maas-ui-68f944f6b6-jh699: Started - Started container maas-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/maas-ui-68f944f6b6-jh699: Created - Created container: maas-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/autorag-ui-58f548cbd-8n44h: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2" in 3.118s (3.118s including waiting). Image size: 223947455 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/automl-ui-74d9b97d49-stzqx: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af" in 4.024s (4.024s including waiting). Image size: 222806700 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/autorag-ui-58f548cbd-8n44h: Started - Started container autorag-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-8dkgk: AddedInterface - Add eth0 [10.134.0.107/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:08 INFO Pod/notebooks-ui-6577cc467-27k2h: Started - Started container notebooks-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Created - Created container: gen-ai-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/notebooks-ui-6577cc467-27k2h: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003" in 4.48s (4.48s including waiting). Image size: 190593254 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987" in 4.114s (4.114s including waiting). Image size: 238860397 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/automl-ui-74d9b97d49-stzqx: Created - Created container: automl-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/automl-ui-74d9b97d49-stzqx: Started - Started container automl-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-95jw9: AddedInterface - Add eth0 [10.132.0.98/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:08 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Started - Started container gen-ai-ui
2026/09/07 21:37:31   21:35:07 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:07 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Started - Started container core-bff
2026/09/07 21:37:31   21:35:07 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Started - Started container kuberay-operator
2026/09/07 21:37:31   21:35:07 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.894s (1.894s including waiting). Image size: 189730715 bytes.
2026/09/07 21:37:31   21:35:07 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Created - Created container: kuberay-operator
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 317ms (317ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:06 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7d87f54798-tk6zl_353039be-4108-4bad-93ca-56d41dd17033 became leader
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: AddedInterface - Add eth0 [10.133.0.117/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 310ms (310ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:06 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3"
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:05 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-858fdb6f4d-wqtjf to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:05 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-pdcnm to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:05 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/07 21:37:31   21:35:05 INFO Pod/autorag-ui-58f548cbd-8n44h: AddedInterface - Add eth0 [10.134.0.105/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:05 INFO Pod/autorag-ui-58f548cbd-8n44h: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2"
2026/09/07 21:37:31   21:35:05 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-858fdb6f4d from 0 to 1
2026/09/07 21:37:31   21:35:05 INFO Pod/kuberay-operator-7d87f54798-pdcnm: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:05 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulCreate - Created pod: rhods-dashboard-858fdb6f4d-wqtjf
2026/09/07 21:37:31   21:35:05 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 1 to 2
2026/09/07 21:37:31   21:35:05 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-pdcnm
2026/09/07 21:37:31   21:35:05 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: AddedInterface - Add eth0 [10.134.0.106/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-57c49dcf65-wdqgz to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:04 INFO Pod/autorag-ui-58f548cbd-8n44h: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-58f548cbd-8n44h to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:04 INFO Pod/automl-ui-74d9b97d49-stzqx: AddedInterface - Add eth0 [10.134.0.104/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-58f548cbd from 0 to 1
2026/09/07 21:37:31   21:35:04 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987"
2026/09/07 21:37:31   21:35:04 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: AddedInterface - Add eth0 [10.134.0.102/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 WARN Pod/agent-ops-ui-57c49dcf65-wdqgz: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "agent-ops-proxy-tls" not found
2026/09/07 21:37:31   21:35:04 INFO Pod/automl-ui-74d9b97d49-stzqx: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af"
2026/09/07 21:37:31   21:35:04 INFO ReplicaSet/autorag-ui-58f548cbd: SuccessfulCreate - Created pod: autorag-ui-58f548cbd-8n44h
2026/09/07 21:37:31   21:35:04 INFO Pod/maas-ui-68f944f6b6-jh699: AddedInterface - Add eth0 [10.134.0.103/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 INFO Pod/maas-ui-68f944f6b6-jh699: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe"
2026/09/07 21:37:31   21:35:04 INFO Pod/notebooks-ui-6577cc467-27k2h: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003"
2026/09/07 21:37:31   21:35:04 INFO Pod/notebooks-ui-6577cc467-27k2h: AddedInterface - Add eth0 [10.134.0.101/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 WARN Pod/autorag-ui-58f548cbd-8n44h: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "autorag-proxy-tls" not found
2026/09/07 21:37:31   21:35:04 INFO ReplicaSet/agent-ops-ui-57c49dcf65: SuccessfulCreate - Created pod: agent-ops-ui-57c49dcf65-wdqgz
2026/09/07 21:37:31   21:35:04 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-57c49dcf65 from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO Pod/automl-ui-74d9b97d49-stzqx: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-74d9b97d49-stzqx to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:03 INFO Pod/maas-ui-68f944f6b6-jh699: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-68f944f6b6-jh699 to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:03 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-c58764dbf-pkh4g to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:03 WARN Pod/notebooks-ui-6577cc467-27k2h: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "notebooks-proxy-tls" not found
2026/09/07 21:37:31   21:35:03 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-68f944f6b6 from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO ReplicaSet/maas-ui-68f944f6b6: SuccessfulCreate - Created pod: maas-ui-68f944f6b6-jh699
2026/09/07 21:37:31   21:35:03 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-74d9b97d49 from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO ReplicaSet/automl-ui-74d9b97d49: SuccessfulCreate - Created pod: automl-ui-74d9b97d49-stzqx
2026/09/07 21:37:31   21:35:03 WARN Pod/gen-ai-ui-c58764dbf-pkh4g: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "gen-ai-proxy-tls" not found
2026/09/07 21:37:31   21:35:03 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-c58764dbf from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO ReplicaSet/gen-ai-ui-c58764dbf: SuccessfulCreate - Created pod: gen-ai-ui-c58764dbf-pkh4g
2026/09/07 21:37:31   21:35:02 INFO Pod/notebooks-ui-6577cc467-27k2h: Scheduled - Successfully assigned redhat-ods-applications/notebooks-ui-6577cc467-27k2h to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 468ms (468ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Deployment/notebooks-ui: ScalingReplicaSet - Scaled up replica set notebooks-ui-6577cc467 from 0 to 1
2026/09/07 21:37:31   21:35:02 INFO ReplicaSet/notebooks-ui-6577cc467: SuccessfulCreate - Created pod: notebooks-ui-6577cc467-27k2h
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 276ms (276ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Started - Started container core-bff
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 308ms (308ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 390ms (390ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Started - Started container core-bff
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-5v54m: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: AddedInterface - Add eth0 [10.132.0.97/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:00 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-558d97b5db-nn7q2 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:00 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-558d97b5db-5v54m to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:00 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/07 21:37:31   21:35:00 WARN Pod/rhods-dashboard-558d97b5db-5v54m: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "dashboard-proxy-tls" not found
2026/09/07 21:37:31   21:35:00 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "dashboard-proxy-tls" not found
2026/09/07 21:37:31   21:35:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-558d97b5db from 0 to 2
2026/09/07 21:37:31   21:35:00 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulCreate - Created pod: rhods-dashboard-558d97b5db-5v54m
2026/09/07 21:37:31   21:35:00 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulCreate - Created pod: rhods-dashboard-558d97b5db-nn7q2
2026/09/07 21:37:31   21:34:59 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Started - Started container manager
2026/09/07 21:37:31   21:34:59 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Started - Started container manager
2026/09/07 21:37:31   21:34:58 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 2.802s (2.802s including waiting). Image size: 239132625 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Created - Created container: manager
2026/09/07 21:37:31   21:34:58 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 2.435s (2.435s including waiting). Image size: 178395693 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 3.172s (3.172s including waiting). Image size: 247667445 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Started - Started container manager
2026/09/07 21:37:31   21:34:58 INFO Pod/model-serving-api-7df8c9655f-k82t5: Created - Created container: server
2026/09/07 21:37:31   21:34:58 INFO Pod/model-serving-api-7df8c9655f-k82t5: Started - Started container server
2026/09/07 21:37:31   21:34:58 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Created - Created container: manager
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 2.543s (2.543s including waiting). Image size: 244249302 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Created - Created container: manager
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Started - Started container manager
2026/09/07 21:37:31   21:34:58 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Created - Created container: manager
2026/09/07 21:37:31   21:34:57 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Created - Created container: manager
2026/09/07 21:37:31   21:34:57 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Started - Started container manager
2026/09/07 21:37:31   21:34:57 INFO Pod/model-serving-api-7df8c9655f-k82t5: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.858s (1.858s including waiting). Image size: 172459788 bytes.
2026/09/07 21:37:31   21:34:57 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 2.072s (2.072s including waiting). Image size: 235279660 bytes.
2026/09/07 21:37:31   21:34:56 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 21:37:31   21:34:56 INFO Pod/odh-model-controller-57457c4c89-nmnhd: AddedInterface - Add eth0 [10.132.0.95/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:56 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-669d6564bd-tkdqm_ded9ae5b-0f64-4208-8f11-54d81a77f251 became leader
2026/09/07 21:37:31   21:34:56 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: AddedInterface - Add eth0 [10.132.0.96/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:56 INFO Pod/model-serving-api-7df8c9655f-k82t5: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 21:37:31   21:34:56 INFO Pod/model-serving-api-7df8c9655f-k82t5: AddedInterface - Add eth0 [10.132.0.94/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:56 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 21:37:31   21:34:56 INFO Pod/ai-gateway-operator-89f69845b-575wm: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:55 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89hz6pl to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:55 WARN Pod/kserve-localmodelnode-agent-8dkgk: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 21:37:31   21:34:55 WARN Pod/kserve-localmodelnode-agent-95jw9: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 21:37:31   21:34:55 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:55 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 21:37:31   21:34:55 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89hz6pl
2026/09/07 21:37:31   21:34:55 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-95jw9
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: AddedInterface - Add eth0 [10.134.0.99/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:55 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 1 to 2
2026/09/07 21:37:31   21:34:55 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-8dkgk
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: AddedInterface - Add eth0 [10.134.0.100/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:55 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: AddedInterface - Add eth0 [10.133.0.114/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 21:37:31   21:34:55 WARN Pod/odh-model-controller-57457c4c89-nmnhd: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found
2026/09/07 21:37:31   21:34:55 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "run-as-ray-user": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": For
bidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 21:37:31   21:34:54 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:54 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-nmnhd to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/model-serving-api-7df8c9655f-k82t5: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-k82t5 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-k7j5f to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-4sgft to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-ph2cn to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-nmnhd
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-k82t5
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-ph2cn
2026/09/07 21:37:31   21:34:54 WARN Pod/model-serving-api-7df8c9655f-k82t5: FailedMount - MountVolume.SetUp failed for volume "tls-certs" : secret "model-serving-api-tls" not found
2026/09/07 21:37:31   21:34:54 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 21:37:31   21:34:54 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 21:37:31   21:34:54 WARN Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "localmodel-webhook-server-cert" not found
2026/09/07 21:37:31   21:34:54 INFO Pod/maas-controller-56c898799-vcwmz: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-4sgft
2026/09/07 21:37:31   21:34:54 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 21:37:31   21:34:54 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-k7j5f
2026/09/07 21:37:31   21:34:54 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 21:37:31   21:34:50 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:49 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: AddedInterface - Add eth0 [10.134.0.98/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Started - Started container kuberay-operator
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Created - Created container: kuberay-operator
2026/09/07 21:37:31   21:34:46 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:45 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-tk6zl to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:45 WARN Pod/kuberay-operator-7d87f54798-tk6zl: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/07 21:37:31   21:34:45 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-5d466b7d66-nknqx_83c7b451-1e26-4e0f-8f4a-d9dbd09c2815 became leader
2026/09/07 21:37:31   21:34:45 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-tk6zl
2026/09/07 21:37:31   21:34:45 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 21:37:31   21:34:44 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:43 INFO Lease/spark-operator-controller-lock: LeaderElection - spark-operator-controller-c594b967f-m4pdj_e1021d78-6bb8-4079-8fb5-69e88540f0e6 became leader
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Started - Started container manager
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Created - Created container: manager
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: AddedInterface - Add eth0 [10.133.0.113/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:39 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:39 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89pvqgj to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:39 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Created - Created container: manager
2026/09/07 21:37:31   21:34:39 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Started - Started container manager
2026/09/07 21:37:31   21:34:39 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89pvqgj
2026/09/07 21:37:31   21:34:39 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 21:37:31   21:34:38 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-tkdqm to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:38 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" already present on machine
2026/09/07 21:37:31   21:34:38 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-tkdqm
2026/09/07 21:37:31   21:34:38 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 0 to 1
2026/09/07 21:37:31   21:34:38 INFO Pod/dashboard-operator-669d6564bd-tkdqm: AddedInterface - Add eth0 [10.134.0.97/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: Created - Created container: nginx
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: AddedInterface - Add eth0 [10.134.0.96/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: AddedInterface - Add eth0 [10.133.0.112/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: Started - Started container nginx
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: Created - Created container: nginx
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: Started - Started container nginx
2026/09/07 21:37:31   21:34:35 INFO Pod/dashboard-redirect-79754c669f-pv77f: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-79754c669f-pv77f to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:35 INFO Pod/dashboard-redirect-79754c669f-xggvp: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-79754c669f-xggvp to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:35 INFO Pod/maas-controller-56c898799-vcwmz: Started - Started container manager
2026/09/07 21:37:31   21:34:35 INFO Pod/maas-controller-56c898799-vcwmz: Created - Created container: manager
2026/09/07 21:37:31   21:34:35 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-79754c669f from 0 to 2
2026/09/07 21:37:31   21:34:35 INFO ReplicaSet/dashboard-redirect-79754c669f: SuccessfulCreate - Created pod: dashboard-redirect-79754c669f-pv77f
2026/09/07 21:37:31   21:34:35 INFO Pod/maas-controller-56c898799-vcwmz: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd" in 501ms (501ms including waiting). Image size: 197531122 bytes.
2026/09/07 21:37:31   21:34:35 INFO ReplicaSet/dashboard-redirect-79754c669f: SuccessfulCreate - Created pod: dashboard-redirect-79754c669f-xggvp
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-5fc6bb8d67-8vkv9 to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:34 INFO Pod/maas-controller-56c898799-vcwmz: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd"
2026/09/07 21:37:31   21:34:34 INFO Pod/maas-controller-56c898799-vcwmz: AddedInterface - Add eth0 [10.133.0.110/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Pulled - Container image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7" already present on machine
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: AddedInterface - Add eth0 [10.133.0.111/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Created - Created container: manager
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Started - Started container manager
2026/09/07 21:37:31   21:34:34 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-5fc6bb8d67 from 0 to 1
2026/09/07 21:37:31   21:34:34 INFO ReplicaSet/feast-operator-controller-manager-5fc6bb8d67: SuccessfulCreate - Created pod: feast-operator-controller-manager-5fc6bb8d67-8vkv9
2026/09/07 21:37:31   21:34:33 INFO Pod/maas-controller-56c898799-vcwmz: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-56c898799-vcwmz to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:33 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-56c898799 from 0 to 1
2026/09/07 21:37:31   21:34:33 INFO ReplicaSet/maas-controller-56c898799: SuccessfulCreate - Created pod: maas-controller-56c898799-vcwmz
2026/09/07 21:37:31   21:34:33 WARN Pod/maas-controller-56c898799-vcwmz: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "maas-controller-webhook-cert" not found
2026/09/07 21:37:31   21:34:32 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:29 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:29 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:28 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:28 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:28 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Started - Started container webhook
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Created - Created container: controller
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Started - Started container controller
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2"
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2" in 583ms (583ms including waiting). Image size: 193888070 bytes.
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Created - Created container: manager
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Started - Started container manager
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Created - Created container: webhook
2026/09/07 21:37:31   21:34:28 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: AddedInterface - Add eth0 [10.133.0.109/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:27 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:27 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-webhook-5d466b7d66-nknqx to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-controller-c594b967f-m4pdj to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: AddedInterface - Add eth0 [10.134.0.95/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 21:37:31   21:34:27 INFO Deployment/spark-operator-controller: ScalingReplicaSet - Scaled up replica set spark-operator-controller-c594b967f from 0 to 1
2026/09/07 21:37:31   21:34:27 INFO ReplicaSet/spark-operator-controller-c594b967f: SuccessfulCreate - Created pod: spark-operator-controller-c594b967f-m4pdj
2026/09/07 21:37:31   21:34:27 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/07 21:37:31   21:34:27 INFO ReplicaSet/ogx-k8s-operator-controller-manager-59777cdbbb: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8
2026/09/07 21:37:31   21:34:27 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/07 21:37:31   21:34:27 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-59777cdbbb from 0 to 1
2026/09/07 21:37:31   21:34:27 INFO ReplicaSet/spark-operator-webhook-5d466b7d66: SuccessfulCreate - Created pod: spark-operator-webhook-5d466b7d66-nknqx
2026/09/07 21:37:31   21:34:27 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:27 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: AddedInterface - Add eth0 [10.132.0.93/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:26 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:24 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:24 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-89f69845b-575wm_5f3bb8f8-c55a-4938-87bf-61beb6f80742 became leader
2026/09/07 21:37:31   21:34:24 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:24 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-mggzt_7922e366-ed21-468f-ba55-50b96b9aa153 became leader
2026/09/07 21:37:31   21:34:24 INFO Lease/opendatahub-feast-operator-lock: LeaderElection - opendatahub-feast-operator-84ff599545-d49rd_527382f3-cc39-422e-b551-f2e4845a1781 became leader
2026/09/07 21:37:31   21:34:21 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-644d45d5b-kvfc5_12f3105b-24c5-421f-ab29-b88a77d95e51 became leader
2026/09/07 21:37:31   21:34:20 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-m4cpw_dc9aa37f-4b29-4d8f-8ab2-5ed6f7209db0 became leader
2026/09/07 21:37:31   21:34:20 INFO Lease/spark-operator-module-controller-manager: LeaderElection - spark-operator-module-controller-manager-59549b5854-gnzt9_f3468754-399c-4ba9-ae79-cd46fa47a561 became leader
2026/09/07 21:37:31   21:34:15 WARN Pod/rhods-dashboard-77f45899f6-84xgj: Unhealthy - Readiness probe failed: Get "https://10.132.0.59:8444/healthz": dial tcp 10.132.0.59:8444: connect: connection refused
2026/09/07 21:37:31   21:34:15 WARN Pod/rhods-dashboard-77f45899f6-84xgj: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:34:15 WARN Pod/rhods-dashboard-77f45899f6-84xgj: Unhealthy - Readiness probe failed: Get "https://10.132.0.59:8943/healthcheck": dial tcp 10.132.0.59:8943: connect: connection refused
2026/09/07 21:37:31   21:34:12 WARN Pod/rhods-dashboard-77f45899f6-zxcz4: Unhealthy - Readiness probe failed: Get "https://10.133.0.50:8444/healthz": dial tcp 10.133.0.50:8444: connect: connection refused
2026/09/07 21:37:31   21:34:10 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728" in 2.514s (2.514s including waiting). Image size: 186574763 bytes.
2026/09/07 21:37:31   21:34:10 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf_362bff1f-57cf-4710-8a24-8e4738bb6ebd became leader
2026/09/07 21:37:31   21:34:10 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Created - Created container: manager
2026/09/07 21:37:31   21:34:10 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Started - Started container manager
2026/09/07 21:37:31   21:34:07 WARN Pod/rhods-dashboard-77f45899f6-zxcz4: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:34:07 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: AddedInterface - Add eth0 [10.132.0.92/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:07 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728"
2026/09/07 21:37:31   21:34:07 WARN Pod/rhods-dashboard-77f45899f6-zxcz4: Unhealthy - Readiness probe failed: Get "https://10.133.0.50:8943/healthcheck": dial tcp 10.133.0.50:8943: connect: connection refused
2026/09/07 21:37:31   21:34:06 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:06 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:06 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:06 INFO Pod/ai-gateway-operator-89f69845b-575wm: Started - Started container manager
2026/09/07 21:37:31   21:34:06 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-5f798b84bf: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf
2026/09/07 21:37:31   21:34:06 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-5f798b84bf from 0 to 1
2026/09/07 21:37:31   21:34:06 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810" in 1.506s (1.506s including waiting). Image size: 173568461 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 284ms (284ms including waiting). Image size: 194988415 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845" in 425ms (425ms including waiting). Image size: 168389130 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845"
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: AddedInterface - Add eth0 [10.134.0.94/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:05 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 21:37:31   21:34:05 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 378ms (378ms including waiting). Image size: 207018016 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/ai-gateway-operator-89f69845b-575wm: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 388ms (389ms including waiting). Image size: 209286205 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Created - Created container: copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 3.577s (3.577s including waiting). Image size: 194988415 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Started - Started container manager
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Started - Started container copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" in 3.629s (3.629s including waiting). Image size: 227888997 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Created - Created container: copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/ai-gateway-operator-89f69845b-575wm: Started - Started container copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/ai-gateway-operator-89f69845b-575wm: Created - Created container: copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 3.596s (3.596s including waiting). Image size: 209286205 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Created - Created container: manager
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Started - Started container copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 3.409s (3.409s including waiting). Image size: 207018016 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Started - Started container manager
2026/09/07 21:37:31   21:34:04 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: AddedInterface - Add eth0 [10.133.0.108/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:04 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810"
2026/09/07 21:37:31   21:34:04 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Created - Created container: manager
2026/09/07 21:37:31   21:34:03 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-59b75488c-h96tw to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Created - Created container: manager
2026/09/07 21:37:31   21:34:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" in 1.501s (1.501s including waiting). Image size: 158342259 bytes.
2026/09/07 21:37:31   21:34:03 WARN Pod/odh-notebook-controller-manager-59b75488c-h96tw: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "odh-notebook-controller-webhook-cert" not found
2026/09/07 21:37:31   21:34:03 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Pulled - Successfully pulled image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637" in 2.693s (2.693s including waiting). Image size: 191954597 bytes.
2026/09/07 21:37:31   21:34:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Started - Started container manager
2026/09/07 21:37:31   21:34:03 INFO ReplicaSet/odh-notebook-controller-manager-59b75488c: SuccessfulCreate - Created pod: odh-notebook-controller-manager-59b75488c-h96tw
2026/09/07 21:37:31   21:34:02 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75c8689979-hnkpd to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:02 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75c8689979 from 0 to 1
2026/09/07 21:37:31   21:34:02 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-59b75488c from 0 to 1
2026/09/07 21:37:31   21:34:02 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: AddedInterface - Add eth0 [10.133.0.107/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:02 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Pulling - Pulling image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35"
2026/09/07 21:37:31   21:34:02 INFO ReplicaSet/notebook-controller-deployment-75c8689979: SuccessfulCreate - Created pod: notebook-controller-deployment-75c8689979-hnkpd
2026/09/07 21:37:31   21:34:01 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:01 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-m4cpw to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:01 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 21:37:31   21:34:01 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/07 21:37:31   21:34:01 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: AddedInterface - Add eth0 [10.132.0.91/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Pulling - Pulling image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637"
2026/09/07 21:37:31   21:34:01 INFO Pod/ai-gateway-operator-89f69845b-575wm: AddedInterface - Add eth0 [10.134.0.90/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Pulling - Pulling image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b"
2026/09/07 21:37:31   21:34:01 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: AddedInterface - Add eth0 [10.134.0.91/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 21:37:31   21:34:01 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-m4cpw
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: AddedInterface - Add eth0 [10.134.0.92/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: AddedInterface - Add eth0 [10.134.0.93/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 21:37:31   21:34:00 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-module-controller-manager-59549b5854-gnzt9 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-kvfc5 to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-d49rd to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-mggzt to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/ai-gateway-operator-89f69845b-575wm: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-575wm to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/spark-operator-module-controller-manager-59549b5854: SuccessfulCreate - Created pod: spark-operator-module-controller-manager-59549b5854-gnzt9
2026/09/07 21:37:31   21:34:00 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO Deployment/spark-operator-module-controller-manager: ScalingReplicaSet - Scaled up replica set spark-operator-module-controller-manager-59549b5854 from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-575wm
2026/09/07 21:37:31   21:34:00 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-mggzt
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-d49rd
2026/09/07 21:37:31   21:34:00 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-kvfc5
2026/09/07 21:37:31   21:34:00 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/07 21:37:31   21:33:59 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Started - Started container manager
2026/09/07 21:37:31   21:33:59 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Created - Created container: manager
2026/09/07 21:37:31   21:33:59 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Started - Started container manager
2026/09/07 21:37:31   21:33:59 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4" in 2.334s (2.334s including waiting). Image size: 143076452 bytes.
2026/09/07 21:37:31   21:33:59 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r_69b6915b-f8dd-475a-b53b-97b65053fd4f became leader
2026/09/07 21:37:31   21:33:59 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Created - Created container: manager
2026/09/07 21:37:31   21:33:58 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" already present on machine
2026/09/07 21:37:31   21:33:56 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-c674d9c9-xc2w6 to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-5vjcn to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:56 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Started - Started container manager
2026/09/07 21:37:31   21:33:56 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-6b668bbbb: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r
2026/09/07 21:37:31   21:33:56 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4"
2026/09/07 21:37:31   21:33:56 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: AddedInterface - Add eth0 [10.134.0.87/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: AddedInterface - Add eth0 [10.134.0.88/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Pulled - Container image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" already present on machine
2026/09/07 21:37:31   21:33:56 INFO Pod/workbenches-operator-c674d9c9-xc2w6: AddedInterface - Add eth0 [10.134.0.89/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:56 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-6b668bbbb from 0 to 1
2026/09/07 21:37:31   21:33:56 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/07 21:37:31   21:33:56 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-5vjcn
2026/09/07 21:37:31   21:33:56 INFO ReplicaSet/workbenches-operator-c674d9c9: SuccessfulCreate - Created pod: workbenches-operator-c674d9c9-xc2w6
2026/09/07 21:37:31   21:33:56 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-c674d9c9 from 0 to 1
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Created - Created container: manager
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: AddedInterface - Add eth0 [10.134.0.86/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Created - Created container: manager
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Pulled - Successfully pulled image "registry.k8s.io/pause:3.9" in 338ms (338ms including waiting). Image size: 750414 bytes.
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Pulling - Pulling image "registry.k8s.io/pause:3.9"
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Started - Started container manager
2026/09/07 21:37:31   21:33:53 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-6b8fb45c8b-lrgkd to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:53 INFO ReplicaSet/feast-operator-controller-manager-6b8fb45c8b: SuccessfulCreate - Created pod: feast-operator-controller-manager-6b8fb45c8b-lrgkd
2026/09/07 21:37:31   21:33:53 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-6b8fb45c8b from 0 to 1
2026/09/07 21:37:31   21:33:51 INFO Pod/dashboard-operator-669d6564bd-zmv4v: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:50 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-f76bn: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/ai-gateway-operator-89f69845b-vrn7g: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/workbenches-operator-c674d9c9-j56wt: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/odh-observability-59db79cb46-fklv9: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/aihub-controller-manager-d877888b8-kdvt2: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/kserve-module-controller-manager-66ff7c977-8qg2c: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:49 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:49 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:49 INFO Pod/opendatahub-feast-operator-84ff599545-q9xzw: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:49 INFO Pod/mlflow-operator-controller-manager-5bbb468f5f-4lrz5: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:48 INFO Pod/spark-operator-module-controller-manager-59549b5854-kxzpn: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:48 INFO Pod/opendatahub-ogx-operator-644d45d5b-ngf9c: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/mlflow-ui-7bb6d46f5f-dh94z: Killing - Stopping container mlflow-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/catalog-controller-manager-55967bdd4b-8psf9: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/maas-ui-68f944f6b6-ph28z: Killing - Stopping container maas-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-kq4s7: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/model-registry-operator-controller-manager-59c8bf9b89-4b76q: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/dashboard-redirect-79754c669f-6vzlx: Killing - Stopping container nginx
2026/09/07 21:37:31   21:33:46 INFO Pod/odh-notebook-controller-manager-5f845bd4cb-6fz56: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/eval-hub-ui-869b89d9b4-glpj6: Killing - Stopping container eval-hub-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/odh-model-controller-6b9dd8b59d-2dggj: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/model-registry-ui-7c777bcbb7-cfggf: Killing - Stopping container model-registry-ui
2026/09/07 21:37:31   21:33:46 WARN Pod/llmisvc-controller-manager-6df7b6c5d4-k658k: Unhealthy - Readiness probe failed: Get "http://10.134.0.52:8081/readyz": dial tcp 10.134.0.52:8081: connect: connection refused
2026/09/07 21:37:31   21:33:46 INFO Pod/notebooks-ui-6577cc467-5gsgg: Killing - Stopping container notebooks-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/dashboard-redirect-79754c669f-skpdj: Killing - Stopping container nginx
2026/09/07 21:37:31   21:33:46 INFO Pod/automl-ui-74d9b97d49-dl67k: Killing - Stopping container automl-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/gen-ai-ui-c58764dbf-j8tl2: Killing - Stopping container gen-ai-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-84xgj: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:33:46 INFO Pod/notebook-controller-deployment-75c8689979-4zdpb: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/agent-ops-ui-57c49dcf65-2qgld: Killing - Stopping container agent-ops-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/autorag-ui-58f548cbd-df2sv: Killing - Stopping container autorag-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-84xgj: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-zxcz4: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-zxcz4: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-zxcz4: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-84xgj: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:33:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f892mnvm: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-5mwrb: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k658k: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/spark-operator-webhook-5d466b7d66-tqjkr: Killing - Stopping container webhook
2026/09/07 21:37:31   21:33:45 INFO Pod/spark-operator-controller-c594b967f-vxjlm: Killing - Stopping container controller
2026/09/07 21:37:31   21:33:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-fsgxh: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/maas-controller-56c898799-tm9z4: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/kuberay-operator-7d87f54798-nnvm6: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:33:45 INFO Pod/kserve-controller-manager-5fbbd6dd88-rtwp8: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/model-serving-api-7df8c9655f-5lmjx: Killing - Stopping container server
2026/09/07 21:37:31   21:33:45 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-g8m9q: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:44 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:44 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:44 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-lj2gq: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 INFO Pod/kserve-localmodelnode-agent-t2kzl: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-nf6kh
2026/09/07 21:37:31   21:33:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-t2kzl
2026/09/07 21:37:31   21:33:42 INFO Pod/kserve-localmodelnode-agent-nf6kh: Killing - Stopping container manager
2026/09/07 21:37:31 === RESOURCE QUOTAS ===
2026/09/07 21:37:31 No resource quotas found in monitored namespaces
2026/09/07 21:37:31 Classification for ConfigMap deletion recovery: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/07 21:37:31   Evidence: event Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get "https://10.133.0.117:8444/healthz": dial tcp 10.133.0.117:8444: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get \"https://10.133.0.117:8444/healthz\": dial tcp 10.133.0.117:8444: connect: connection refused"],"confidence":"medium"}
2026/09/07 21:37:31 Diagnostics complete for failed test: ConfigMap deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/07 21:37:31 TEST FAILURE DETECTED: Validate resource deletion recovery
2026/09/07 21:37:31 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:Validate resource deletion recovery) ===
2026/09/07 21:37:31 === CLUSTER STATE ===
2026/09/07 21:37:31 Node: ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:37:31   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:37:31   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:37:31   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:37:31   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:37:31 Node: ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:37:31   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:37:31   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:37:31   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:37:31   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:37:31 Node: ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:37:31   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:37:31   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:37:31   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:37:31   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:37:31 === DEPLOYMENTS ===
2026/09/07 21:37:31 Namespace: redhat-ods-applications
2026/09/07 21:37:31   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 21:37:31   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 21:37:31   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment kserve-controller-manager: 2/2 (READY)
2026/09/07 21:37:31   Deployment kserve-localmodel-controller-manager: 2/2 (READY)
2026/09/07 21:37:31   Deployment kserve-module-controller-manager: 1/1 (READY)
2026/09/07 21:37:31   Deployment llmisvc-controller-manager: 2/2 (READY)
2026/09/07 21:37:31   Deployment maas-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment model-serving-api: 2/2 (READY)
2026/09/07 21:37:31   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 21:37:31   Deployment odh-model-controller: 2/2 (READY)
2026/09/07 21:37:31   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 21:37:31 Namespace: kube-system
2026/09/07 21:37:31 === PODS ===
2026/09/07 21:37:31 Failed to collect pod data: redhat-ods-applications/rhods-dashboard-858fdb6f4d-wqtjf: container kube-rbac-proxy not ready
2026/09/07 21:37:31 Continuing with partial pod data to collect logs from problematic containers
2026/09/07 21:37:31 Namespace: redhat-ods-applications
2026/09/07 21:37:31   Pod rhods-dashboard-858fdb6f4d-wqtjf: Running
2026/09/07 21:37:31     Container kube-rbac-proxy: ready=false
2026/09/07 21:37:31 Namespace kube-system: no problematic pods found
2026/09/07 21:37:31 === OPERATOR STATUS ===
2026/09/07 21:37:31 Operator deployment: 3/3 ready
2026/09/07 21:37:31 === DSCI STATUS ===
2026/09/07 21:37:31 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 21:37:31 === DSC STATUS ===
2026/09/07 21:37:31 DSC default-dsc:
2026/09/07 21:37:31   Ready: True - 
2026/09/07 21:37:31   ProvisioningSucceeded: True - 
2026/09/07 21:37:31   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 21:37:31   ModulesReady: True - 
2026/09/07 21:37:31   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 21:37:31   DashboardReady: True - 
2026/09/07 21:37:31   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   KserveLLMInferenceServiceDependencies: True - 
2026/09/07 21:37:31   KserveLLMInferenceServiceWideEPDependencies: True - 
2026/09/07 21:37:31   KserveReady: True - 
2026/09/07 21:37:31   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 21:37:31   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   RayReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   TrainerReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 21:37:31   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 21:37:31   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 21:37:31 === RECENT EVENTS ===
2026/09/07 21:37:31   21:37:29 WARN Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get "https://10.133.0.117:8444/healthz": dial tcp 10.133.0.117:8444: connect: connection refused
2026/09/07 21:37:31   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:37:17 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulDelete - Deleted pod: rhods-dashboard-858fdb6f4d-wqtjf
2026/09/07 21:37:31   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:37:17 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-858fdb6f4d from 1 to 0
2026/09/07 21:37:31   21:37:05 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe failed: Get "https://10.132.0.97:8444/healthz": dial tcp 10.132.0.97:8444: connect: connection refused
2026/09/07 21:37:31   21:36:58 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Started - Started container manager
2026/09/07 21:37:31   21:36:57 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Created - Created container: manager
2026/09/07 21:37:31   21:36:57 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 1.904s (1.904s including waiting). Image size: 239132625 bytes.
2026/09/07 21:37:31   21:36:56 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 21:37:31   21:36:55 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-xg8fl to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:36:55 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/07 21:37:31   21:36:55 INFO Pod/odh-model-controller-57457c4c89-xg8fl: AddedInterface - Add eth0 [10.134.0.112/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:55 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-xg8fl
2026/09/07 21:37:31   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 354ms (354ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: core-bff
2026/09/07 21:37:31   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container core-bff
2026/09/07 21:37:31   21:36:40 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:36:40 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe failed: Get "https://10.132.0.97:8943/healthcheck": dial tcp 10.132.0.97:8943: connect: connection refused
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 416ms (416ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: AddedInterface - Add eth0 [10.132.0.102/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-764dcdbfcc-4chfj to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:36:38 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-558d97b5db from 1 to 0
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:36:38 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-764dcdbfcc from 1 to 2
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:36:38 INFO ReplicaSet/rhods-dashboard-764dcdbfcc: SuccessfulCreate - Created pod: rhods-dashboard-764dcdbfcc-4chfj
2026/09/07 21:37:31   21:36:38 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulDelete - Deleted pod: rhods-dashboard-558d97b5db-nn7q2
2026/09/07 21:37:31   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.42s (1.42s including waiting). Image size: 172459788 bytes.
2026/09/07 21:37:31   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Started - Started container server
2026/09/07 21:37:31   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Created - Created container: server
2026/09/07 21:37:31   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-xfmcz to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:36:35 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/07 21:37:31   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: AddedInterface - Add eth0 [10.134.0.111/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 21:37:31   21:36:35 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-xfmcz
2026/09/07 21:37:31   21:36:30 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:36:30 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8943/healthcheck": dial tcp 10.132.0.101:8943: connect: connection refused
2026/09/07 21:37:31   21:36:27 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
2026/09/07 21:37:31   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 329ms (329ms including waiting). Image size: 235279660 bytes.
2026/09/07 21:37:31   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Started - Started container manager
2026/09/07 21:37:31   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Created - Created container: manager
2026/09/07 21:37:31   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-j2nvj to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:36:15 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/07 21:37:31   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: AddedInterface - Add eth0 [10.134.0.110/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 21:37:31   21:36:15 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-j2nvj
2026/09/07 21:37:31   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe failed: Get "https://10.133.0.115:8943/healthcheck": dial tcp 10.133.0.115:8943: connect: connection refused
2026/09/07 21:37:31   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe failed: Get "https://10.133.0.115:8444/healthz": dial tcp 10.133.0.115:8444: connect: connection refused
2026/09/07 21:37:31   21:36:05 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:05 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:04 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 400ms (400ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 324ms (324ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: core-bff
2026/09/07 21:37:31   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container core-bff
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-764dcdbfcc-lt7q7 to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: AddedInterface - Add eth0 [10.133.0.119/23] from ovn-kubernetes
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:36:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-764dcdbfcc from 0 to 1
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:36:00 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulDelete - Deleted pod: rhods-dashboard-858fdb6f4d-22p4c
2026/09/07 21:37:31   21:36:00 INFO ReplicaSet/rhods-dashboard-764dcdbfcc: SuccessfulCreate - Created pod: rhods-dashboard-764dcdbfcc-lt7q7
2026/09/07 21:37:31   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:36:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-858fdb6f4d from 2 to 1
2026/09/07 21:37:31   21:35:59 INFO Pod/automl-ui-74d9b97d49-stzqx: Killing - Stopping container automl-ui
2026/09/07 21:37:31   21:35:59 INFO Pod/autorag-ui-58f548cbd-8n44h: Killing - Stopping container autorag-ui
2026/09/07 21:37:31   21:35:59 WARN Endpoints/odh-dashboard-autorag-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-autorag-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-autorag-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-autorag-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: c737a2c7-49fc-4ca4-a807-a07ef76ed351, UID in object meta: 
2026/09/07 21:37:31   21:35:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 349ms (349ms including waiting). Image size: 244249302 bytes.
2026/09/07 21:37:31   21:35:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Created - Created container: manager
2026/09/07 21:37:31   21:35:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Started - Started container manager
2026/09/07 21:37:31   21:35:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-lg2tv to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:55 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:35:55 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-lg2tv
2026/09/07 21:37:31   21:35:55 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 1 to 2
2026/09/07 21:37:31   21:35:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 21:37:31   21:35:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: AddedInterface - Add eth0 [10.133.0.118/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:51 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:51 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:51 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:46 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:46 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Started - Started container core-bff
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 362ms (362ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 347ms (347ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:45 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:44 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:44 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-858fdb6f4d-22p4c to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:44 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-558d97b5db from 2 to 1
2026/09/07 21:37:31   21:35:44 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulCreate - Created pod: rhods-dashboard-858fdb6f4d-22p4c
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:35:44 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Killing - Stopping container webhook
2026/09/07 21:37:31   21:35:44 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: AddedInterface - Add eth0 [10.132.0.101/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:44 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Killing - Stopping container controller
2026/09/07 21:37:31   21:35:44 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-858fdb6f4d from 1 to 2
2026/09/07 21:37:31   21:35:44 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulDelete - Deleted pod: rhods-dashboard-558d97b5db-5v54m
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Created - Created container: kuberay-operator
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Started - Started container kuberay-operator
2026/09/07 21:37:31   21:35:41 INFO Pod/kuberay-operator-7d87f54798-4k8xx: AddedInterface - Add eth0 [10.134.0.109/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:40 INFO Pod/kuberay-operator-7d87f54798-4k8xx: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-4k8xx to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:40 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:35:40 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-4k8xx
2026/09/07 21:37:31   21:35:40 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 21:37:31   21:35:40 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:35:38 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 1.246s (1.246s including waiting). Image size: 178395693 bytes.
2026/09/07 21:37:31   21:35:38 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Started - Started container manager
2026/09/07 21:37:31   21:35:38 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Created - Created container: manager
2026/09/07 21:37:31   21:35:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Created - Created container: manager
2026/09/07 21:37:31   21:35:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 1.871s (1.871s including waiting). Image size: 247667445 bytes.
2026/09/07 21:37:31   21:35:37 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Started - Started container manager
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f8997dhh to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Killing - Stopping container manager
2026/09/07 21:37:31   21:35:36 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f8997dhh
2026/09/07 21:37:31   21:35:36 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f8997dhh: AddedInterface - Add eth0 [10.134.0.108/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:36 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 21:37:31   21:35:35 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-n8xx4 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:35 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-n8xx4
2026/09/07 21:37:31   21:35:35 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: AddedInterface - Add eth0 [10.132.0.100/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:35 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 21:37:31   21:35:35 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 1 to 2
2026/09/07 21:37:31   21:35:17 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Started - Started container manager
2026/09/07 21:37:31   21:35:17 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" in 1.42s (1.42s including waiting). Image size: 170934264 bytes.
2026/09/07 21:37:31   21:35:17 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Created - Created container: manager
2026/09/07 21:37:31   21:35:16 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-nmnhd_91fbcd1f-6e9c-4b7e-a296-8a92e6a4eed0 became leader
2026/09/07 21:37:31   21:35:15 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-6qzxp to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:15 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 1 to 2
2026/09/07 21:37:31   21:35:15 INFO Pod/dashboard-operator-669d6564bd-6qzxp: AddedInterface - Add eth0 [10.132.0.99/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:15 INFO Pod/dashboard-operator-669d6564bd-6qzxp: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520"
2026/09/07 21:37:31   21:35:15 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-ph2cn_856dd34e-3152-48ff-9d1c-a5c8dba42773 became leader
2026/09/07 21:37:31   21:35:15 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-6qzxp
2026/09/07 21:37:31   21:35:14 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-k7j5f_d06a40c7-50a7-4800-9783-43c5f10727a0 became leader
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Created - Created container: manager
2026/09/07 21:37:31   21:35:09 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3" in 3.054s (3.055s including waiting). Image size: 184646446 bytes.
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Started - Started container manager
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 385ms (385ms including waiting). Image size: 244000959 bytes.
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf" in 323ms (323ms including waiting). Image size: 244000959 bytes.
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-8dkgk: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodelnode-agent-rhel9@sha256:b3f9fe59c8ca861bc80b0338a1baeb03e765734ed358d7098ddbaf1fd917d1bf"
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Created - Created container: manager
2026/09/07 21:37:31   21:35:09 INFO Pod/kserve-localmodelnode-agent-95jw9: Started - Started container manager
2026/09/07 21:37:31   21:35:09 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Created - Created container: agent-ops-ui
2026/09/07 21:37:31   21:35:09 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Started - Started container agent-ops-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-95jw9: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-95jw9 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-8dkgk: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodelnode-agent-8dkgk to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:08 INFO Pod/autorag-ui-58f548cbd-8n44h: Created - Created container: autorag-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/maas-ui-68f944f6b6-jh699: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe" in 4.366s (4.366s including waiting). Image size: 189581000 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/notebooks-ui-6577cc467-27k2h: Created - Created container: notebooks-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/maas-ui-68f944f6b6-jh699: Started - Started container maas-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/maas-ui-68f944f6b6-jh699: Created - Created container: maas-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/autorag-ui-58f548cbd-8n44h: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2" in 3.118s (3.118s including waiting). Image size: 223947455 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/automl-ui-74d9b97d49-stzqx: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af" in 4.024s (4.024s including waiting). Image size: 222806700 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/autorag-ui-58f548cbd-8n44h: Started - Started container autorag-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-8dkgk: AddedInterface - Add eth0 [10.134.0.107/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:08 INFO Pod/notebooks-ui-6577cc467-27k2h: Started - Started container notebooks-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Created - Created container: gen-ai-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/notebooks-ui-6577cc467-27k2h: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003" in 4.48s (4.48s including waiting). Image size: 190593254 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Pulled - Successfully pulled image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987" in 4.114s (4.114s including waiting). Image size: 238860397 bytes.
2026/09/07 21:37:31   21:35:08 INFO Pod/automl-ui-74d9b97d49-stzqx: Created - Created container: automl-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/automl-ui-74d9b97d49-stzqx: Started - Started container automl-ui
2026/09/07 21:37:31   21:35:08 INFO Pod/kserve-localmodelnode-agent-95jw9: AddedInterface - Add eth0 [10.132.0.98/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:08 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Started - Started container gen-ai-ui
2026/09/07 21:37:31   21:35:07 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:07 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Started - Started container core-bff
2026/09/07 21:37:31   21:35:07 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Started - Started container kuberay-operator
2026/09/07 21:37:31   21:35:07 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Pulled - Successfully pulled image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" in 1.894s (1.894s including waiting). Image size: 189730715 bytes.
2026/09/07 21:37:31   21:35:07 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Created - Created container: kuberay-operator
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 317ms (317ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:06 INFO Lease/ray-operator-leader: LeaderElection - kuberay-operator-7d87f54798-tk6zl_353039be-4108-4bad-93ca-56d41dd17033 became leader
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: AddedInterface - Add eth0 [10.133.0.117/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 310ms (310ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:06 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-agent-ops-rhel9@sha256:0e5ffb00bed6b5563f407fc668c6c08875751d175f5442ffaf5440dfa27068c3"
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:06 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:05 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-858fdb6f4d-wqtjf to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:05 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-pdcnm to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:05 INFO Pod/kuberay-operator-7d87f54798-pdcnm: Pulling - Pulling image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22"
2026/09/07 21:37:31   21:35:05 INFO Pod/autorag-ui-58f548cbd-8n44h: AddedInterface - Add eth0 [10.134.0.105/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:05 INFO Pod/autorag-ui-58f548cbd-8n44h: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-autorag-rhel9@sha256:92738668ed5951a6ee336b3a72af14c491499c2a67e59b7a648678b2f73e78e2"
2026/09/07 21:37:31   21:35:05 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-858fdb6f4d from 0 to 1
2026/09/07 21:37:31   21:35:05 INFO Pod/kuberay-operator-7d87f54798-pdcnm: AddedInterface - Add eth0 [10.133.0.116/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:05 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulCreate - Created pod: rhods-dashboard-858fdb6f4d-wqtjf
2026/09/07 21:37:31   21:35:05 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 1 to 2
2026/09/07 21:37:31   21:35:05 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-pdcnm
2026/09/07 21:37:31   21:35:05 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: AddedInterface - Add eth0 [10.134.0.106/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 INFO Pod/agent-ops-ui-57c49dcf65-wdqgz: Scheduled - Successfully assigned redhat-ods-applications/agent-ops-ui-57c49dcf65-wdqgz to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:04 INFO Pod/autorag-ui-58f548cbd-8n44h: Scheduled - Successfully assigned redhat-ods-applications/autorag-ui-58f548cbd-8n44h to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:04 INFO Pod/automl-ui-74d9b97d49-stzqx: AddedInterface - Add eth0 [10.134.0.104/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 INFO Deployment/autorag-ui: ScalingReplicaSet - Scaled up replica set autorag-ui-58f548cbd from 0 to 1
2026/09/07 21:37:31   21:35:04 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-gen-ai-rhel9@sha256:2c0cf52936b70ce79f10dbdd631c7a5173458d70170e27747983fba6ca35c987"
2026/09/07 21:37:31   21:35:04 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: AddedInterface - Add eth0 [10.134.0.102/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 WARN Pod/agent-ops-ui-57c49dcf65-wdqgz: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "agent-ops-proxy-tls" not found
2026/09/07 21:37:31   21:35:04 INFO Pod/automl-ui-74d9b97d49-stzqx: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-automl-rhel9@sha256:9206030f69d39f930b941544407e17ddd64549e82e37aa50e6b40d15dd0242af"
2026/09/07 21:37:31   21:35:04 INFO ReplicaSet/autorag-ui-58f548cbd: SuccessfulCreate - Created pod: autorag-ui-58f548cbd-8n44h
2026/09/07 21:37:31   21:35:04 INFO Pod/maas-ui-68f944f6b6-jh699: AddedInterface - Add eth0 [10.134.0.103/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 INFO Pod/maas-ui-68f944f6b6-jh699: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-maas-rhel9@sha256:90dcc97113187d4f1f54cecfb7ef38cd42b4dbd29f3dbcc769c296a21f61bbfe"
2026/09/07 21:37:31   21:35:04 INFO Pod/notebooks-ui-6577cc467-27k2h: Pulling - Pulling image "quay.io/rhoai/odh-mod-arch-notebooks-rhel9@sha256:5db20dded9347c00cc21be83b68ad5d7cd7cf0458620d3013443507b94828003"
2026/09/07 21:37:31   21:35:04 INFO Pod/notebooks-ui-6577cc467-27k2h: AddedInterface - Add eth0 [10.134.0.101/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:04 WARN Pod/autorag-ui-58f548cbd-8n44h: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "autorag-proxy-tls" not found
2026/09/07 21:37:31   21:35:04 INFO ReplicaSet/agent-ops-ui-57c49dcf65: SuccessfulCreate - Created pod: agent-ops-ui-57c49dcf65-wdqgz
2026/09/07 21:37:31   21:35:04 INFO Deployment/agent-ops-ui: ScalingReplicaSet - Scaled up replica set agent-ops-ui-57c49dcf65 from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO Pod/automl-ui-74d9b97d49-stzqx: Scheduled - Successfully assigned redhat-ods-applications/automl-ui-74d9b97d49-stzqx to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:03 INFO Pod/maas-ui-68f944f6b6-jh699: Scheduled - Successfully assigned redhat-ods-applications/maas-ui-68f944f6b6-jh699 to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:03 INFO Pod/gen-ai-ui-c58764dbf-pkh4g: Scheduled - Successfully assigned redhat-ods-applications/gen-ai-ui-c58764dbf-pkh4g to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:03 WARN Pod/notebooks-ui-6577cc467-27k2h: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "notebooks-proxy-tls" not found
2026/09/07 21:37:31   21:35:03 INFO Deployment/maas-ui: ScalingReplicaSet - Scaled up replica set maas-ui-68f944f6b6 from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO ReplicaSet/maas-ui-68f944f6b6: SuccessfulCreate - Created pod: maas-ui-68f944f6b6-jh699
2026/09/07 21:37:31   21:35:03 INFO Deployment/automl-ui: ScalingReplicaSet - Scaled up replica set automl-ui-74d9b97d49 from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO ReplicaSet/automl-ui-74d9b97d49: SuccessfulCreate - Created pod: automl-ui-74d9b97d49-stzqx
2026/09/07 21:37:31   21:35:03 WARN Pod/gen-ai-ui-c58764dbf-pkh4g: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "gen-ai-proxy-tls" not found
2026/09/07 21:37:31   21:35:03 INFO Deployment/gen-ai-ui: ScalingReplicaSet - Scaled up replica set gen-ai-ui-c58764dbf from 0 to 1
2026/09/07 21:37:31   21:35:03 INFO ReplicaSet/gen-ai-ui-c58764dbf: SuccessfulCreate - Created pod: gen-ai-ui-c58764dbf-pkh4g
2026/09/07 21:37:31   21:35:02 INFO Pod/notebooks-ui-6577cc467-27k2h: Scheduled - Successfully assigned redhat-ods-applications/notebooks-ui-6577cc467-27k2h to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Created - Created container: rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 468ms (468ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Deployment/notebooks-ui: ScalingReplicaSet - Scaled up replica set notebooks-ui-6577cc467 from 0 to 1
2026/09/07 21:37:31   21:35:02 INFO ReplicaSet/notebooks-ui-6577cc467: SuccessfulCreate - Created pod: notebooks-ui-6577cc467-27k2h
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 276ms (276ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Started - Started container core-bff
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Created - Created container: core-bff
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 308ms (308ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Created - Created container: kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Started - Started container rhods-dashboard
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 390ms (390ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Started - Started container kube-rbac-proxy
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:37:31   21:35:02 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Started - Started container core-bff
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-5v54m: AddedInterface - Add eth0 [10.133.0.115/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:37:31   21:35:01 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: AddedInterface - Add eth0 [10.132.0.97/23] from ovn-kubernetes
2026/09/07 21:37:31   21:35:00 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-558d97b5db-nn7q2 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:35:00 INFO Pod/rhods-dashboard-558d97b5db-5v54m: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-558d97b5db-5v54m to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:35:00 INFO PodDisruptionBudget/rhods-dashboard: NoPods - No matching pods found
2026/09/07 21:37:31   21:35:00 WARN Pod/rhods-dashboard-558d97b5db-5v54m: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "dashboard-proxy-tls" not found
2026/09/07 21:37:31   21:35:00 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: FailedMount - MountVolume.SetUp failed for volume "proxy-tls" : secret "dashboard-proxy-tls" not found
2026/09/07 21:37:31   21:35:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-558d97b5db from 0 to 2
2026/09/07 21:37:31   21:35:00 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulCreate - Created pod: rhods-dashboard-558d97b5db-5v54m
2026/09/07 21:37:31   21:35:00 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulCreate - Created pod: rhods-dashboard-558d97b5db-nn7q2
2026/09/07 21:37:31   21:34:59 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Started - Started container manager
2026/09/07 21:37:31   21:34:59 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Started - Started container manager
2026/09/07 21:37:31   21:34:58 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 2.802s (2.802s including waiting). Image size: 239132625 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Created - Created container: manager
2026/09/07 21:37:31   21:34:58 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Pulled - Successfully pulled image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" in 2.435s (2.435s including waiting). Image size: 178395693 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 3.172s (3.172s including waiting). Image size: 247667445 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Started - Started container manager
2026/09/07 21:37:31   21:34:58 INFO Pod/model-serving-api-7df8c9655f-k82t5: Created - Created container: server
2026/09/07 21:37:31   21:34:58 INFO Pod/model-serving-api-7df8c9655f-k82t5: Started - Started container server
2026/09/07 21:37:31   21:34:58 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Created - Created container: manager
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 2.543s (2.543s including waiting). Image size: 244249302 bytes.
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Created - Created container: manager
2026/09/07 21:37:31   21:34:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Started - Started container manager
2026/09/07 21:37:31   21:34:58 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Created - Created container: manager
2026/09/07 21:37:31   21:34:57 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Created - Created container: manager
2026/09/07 21:37:31   21:34:57 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Started - Started container manager
2026/09/07 21:37:31   21:34:57 INFO Pod/model-serving-api-7df8c9655f-k82t5: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.858s (1.858s including waiting). Image size: 172459788 bytes.
2026/09/07 21:37:31   21:34:57 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 2.072s (2.072s including waiting). Image size: 235279660 bytes.
2026/09/07 21:37:31   21:34:56 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 21:37:31   21:34:56 INFO Pod/odh-model-controller-57457c4c89-nmnhd: AddedInterface - Add eth0 [10.132.0.95/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:56 INFO Lease/dashboard.components.platform.opendatahub.io: LeaderElection - dashboard-operator-669d6564bd-tkdqm_ded9ae5b-0f64-4208-8f11-54d81a77f251 became leader
2026/09/07 21:37:31   21:34:56 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: AddedInterface - Add eth0 [10.132.0.96/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:56 INFO Pod/model-serving-api-7df8c9655f-k82t5: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 21:37:31   21:34:56 INFO Pod/model-serving-api-7df8c9655f-k82t5: AddedInterface - Add eth0 [10.132.0.94/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:56 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Pulling - Pulling image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e"
2026/09/07 21:37:31   21:34:56 INFO Pod/ai-gateway-operator-89f69845b-575wm: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:55 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89hz6pl: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89hz6pl to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:55 WARN Pod/kserve-localmodelnode-agent-8dkgk: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 21:37:31   21:34:55 WARN Pod/kserve-localmodelnode-agent-95jw9: FailedScheduling - 0/3 nodes are available: pod has unbound immediate PersistentVolumeClaims. not found
2026/09/07 21:37:31   21:34:55 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:55 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 21:37:31   21:34:55 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89hz6pl
2026/09/07 21:37:31   21:34:55 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-95jw9
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: AddedInterface - Add eth0 [10.134.0.99/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:55 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 1 to 2
2026/09/07 21:37:31   21:34:55 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulCreate - Created pod: kserve-localmodelnode-agent-8dkgk
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: AddedInterface - Add eth0 [10.134.0.100/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:55 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: AddedInterface - Add eth0 [10.133.0.114/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 21:37:31   21:34:55 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 21:37:31   21:34:55 WARN Pod/odh-model-controller-57457c4c89-nmnhd: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found
2026/09/07 21:37:31   21:34:55 WARN DaemonSet/kserve-localmodelnode-agent: FailedCreate - Error creating: pods "kserve-localmodelnode-agent-" is forbidden: unable to validate against any security context constraint: [provider "anyuid": Forbidden: not usable by user or serviceaccount, provider "run-as-ray-user": Forbidden: not usable by user or serviceaccount, provider restricted-v2: .spec.securityContext.fsGroup: Invalid value: [1000]: 1000 is not an allowed group, provider restricted-v2: .containers[0].runAsUser: Invalid value: 1000: must be in the ranges: [1000620000, 1000629999], provider restricted-v3: .spec.securityContext.hostUsers: Invalid value: null: Host Users must be set to false, provider "restricted": Forbidden: not usable by user or serviceaccount, provider "nested-container": Forbidden: not usable by user or serviceaccount, provider "nonroot-v2": Forbidden: not usable by user or serviceaccount, provider "nonroot": Forbidden: not usable by user or serviceaccount, provider "hostmount-anyuid": For
bidden: not usable by user or serviceaccount, provider "hostmount-anyuid-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork-v2": Forbidden: not usable by user or serviceaccount, provider "hostnetwork": Forbidden: not usable by user or serviceaccount, provider "hostaccess": Forbidden: not usable by user or serviceaccount, provider "insights-runtime-extractor-scc": Forbidden: not usable by user or serviceaccount, provider "node-exporter": Forbidden: not usable by user or serviceaccount, provider "privileged": Forbidden: not usable by user or serviceaccount]
2026/09/07 21:37:31   21:34:54 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:54 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-nmnhd to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/model-serving-api-7df8c9655f-k82t5: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-k82t5 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-k7j5f to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-4sgft to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-ph2cn to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-nmnhd
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-k82t5
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-ph2cn
2026/09/07 21:37:31   21:34:54 WARN Pod/model-serving-api-7df8c9655f-k82t5: FailedMount - MountVolume.SetUp failed for volume "tls-certs" : secret "model-serving-api-tls" not found
2026/09/07 21:37:31   21:34:54 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 21:37:31   21:34:54 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 21:37:31   21:34:54 WARN Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "localmodel-webhook-server-cert" not found
2026/09/07 21:37:31   21:34:54 INFO Pod/maas-controller-56c898799-vcwmz: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-4sgft
2026/09/07 21:37:31   21:34:54 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 21:37:31   21:34:54 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 21:37:31   21:34:54 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-k7j5f
2026/09/07 21:37:31   21:34:54 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 21:37:31   21:34:50 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:49 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: AddedInterface - Add eth0 [10.134.0.98/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Started - Started container kuberay-operator
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Pulled - Container image "quay.io/rhoai/odh-kuberay-operator-controller-rhel9@sha256:e4365e4fa174bf2f38fa4a737ac270585afe034bc89b859b735118b08e89bf22" already present on machine
2026/09/07 21:37:31   21:34:46 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Created - Created container: kuberay-operator
2026/09/07 21:37:31   21:34:46 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:45 INFO Pod/kuberay-operator-7d87f54798-tk6zl: Scheduled - Successfully assigned redhat-ods-applications/kuberay-operator-7d87f54798-tk6zl to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:45 WARN Pod/kuberay-operator-7d87f54798-tk6zl: FailedMount - MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found
2026/09/07 21:37:31   21:34:45 INFO Lease/spark-operator-webhook-lock: LeaderElection - spark-operator-webhook-5d466b7d66-nknqx_83c7b451-1e26-4e0f-8f4a-d9dbd09c2815 became leader
2026/09/07 21:37:31   21:34:45 INFO ReplicaSet/kuberay-operator-7d87f54798: SuccessfulCreate - Created pod: kuberay-operator-7d87f54798-tk6zl
2026/09/07 21:37:31   21:34:45 INFO Deployment/kuberay-operator: ScalingReplicaSet - Scaled up replica set kuberay-operator-7d87f54798 from 0 to 1
2026/09/07 21:37:31   21:34:44 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:43 INFO Lease/spark-operator-controller-lock: LeaderElection - spark-operator-controller-c594b967f-m4pdj_e1021d78-6bb8-4079-8fb5-69e88540f0e6 became leader
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Started - Started container manager
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Created - Created container: manager
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Pulled - Container image "quay.io/rhoai/odh-data-science-pipelines-operator-controller-rhel9@sha256:f5c36e87054387a8f630e5b0ada31fdac2f5e39d547dc2a28b890b9504f23a6e" already present on machine
2026/09/07 21:37:31   21:34:40 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: AddedInterface - Add eth0 [10.133.0.113/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:39 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:39 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f89pvqgj: Scheduled - Successfully assigned redhat-ods-applications/data-science-pipelines-operator-controller-manager-56b8f89pvqgj to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:39 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Created - Created container: manager
2026/09/07 21:37:31   21:34:39 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Started - Started container manager
2026/09/07 21:37:31   21:34:39 INFO ReplicaSet/data-science-pipelines-operator-controller-manager-56b8f89dc6: SuccessfulCreate - Created pod: data-science-pipelines-operator-controller-manager-56b8f89pvqgj
2026/09/07 21:37:31   21:34:39 INFO Deployment/data-science-pipelines-operator-controller-manager: ScalingReplicaSet - Scaled up replica set data-science-pipelines-operator-controller-manager-56b8f89dc6 from 0 to 1
2026/09/07 21:37:31   21:34:38 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Scheduled - Successfully assigned redhat-ods-applications/dashboard-operator-669d6564bd-tkdqm to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:38 INFO Pod/dashboard-operator-669d6564bd-tkdqm: Pulled - Container image "quay.io/rhoai/odh-dashboard-operator-rhel9@sha256:24577ce8b7974dfe2077537471a01e0be0660fa9189d58ee40f8cc5d1fb73520" already present on machine
2026/09/07 21:37:31   21:34:38 INFO ReplicaSet/dashboard-operator-669d6564bd: SuccessfulCreate - Created pod: dashboard-operator-669d6564bd-tkdqm
2026/09/07 21:37:31   21:34:38 INFO Deployment/dashboard-operator: ScalingReplicaSet - Scaled up replica set dashboard-operator-669d6564bd from 0 to 1
2026/09/07 21:37:31   21:34:38 INFO Pod/dashboard-operator-669d6564bd-tkdqm: AddedInterface - Add eth0 [10.134.0.97/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: Created - Created container: nginx
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: AddedInterface - Add eth0 [10.134.0.96/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: AddedInterface - Add eth0 [10.133.0.112/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-pv77f: Started - Started container nginx
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: Pulled - Container image "registry.redhat.io/ubi9/nginx-126@sha256:e8eb9cf10f384ab58cca89757484ec8d29db4e90f4eaad363d06b1253215127d" already present on machine
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: Created - Created container: nginx
2026/09/07 21:37:31   21:34:36 INFO Pod/dashboard-redirect-79754c669f-xggvp: Started - Started container nginx
2026/09/07 21:37:31   21:34:35 INFO Pod/dashboard-redirect-79754c669f-pv77f: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-79754c669f-pv77f to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:35 INFO Pod/dashboard-redirect-79754c669f-xggvp: Scheduled - Successfully assigned redhat-ods-applications/dashboard-redirect-79754c669f-xggvp to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:35 INFO Pod/maas-controller-56c898799-vcwmz: Started - Started container manager
2026/09/07 21:37:31   21:34:35 INFO Pod/maas-controller-56c898799-vcwmz: Created - Created container: manager
2026/09/07 21:37:31   21:34:35 INFO Deployment/dashboard-redirect: ScalingReplicaSet - Scaled up replica set dashboard-redirect-79754c669f from 0 to 2
2026/09/07 21:37:31   21:34:35 INFO ReplicaSet/dashboard-redirect-79754c669f: SuccessfulCreate - Created pod: dashboard-redirect-79754c669f-pv77f
2026/09/07 21:37:31   21:34:35 INFO Pod/maas-controller-56c898799-vcwmz: Pulled - Successfully pulled image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd" in 501ms (501ms including waiting). Image size: 197531122 bytes.
2026/09/07 21:37:31   21:34:35 INFO ReplicaSet/dashboard-redirect-79754c669f: SuccessfulCreate - Created pod: dashboard-redirect-79754c669f-xggvp
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-5fc6bb8d67-8vkv9 to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:34 INFO Pod/maas-controller-56c898799-vcwmz: Pulling - Pulling image "quay.io/rhoai/odh-maas-controller-rhel9@sha256:b9f043f3d4df543b20efeb5c07f195d9b597323913e8aa99d94151fc82ed70dd"
2026/09/07 21:37:31   21:34:34 INFO Pod/maas-controller-56c898799-vcwmz: AddedInterface - Add eth0 [10.133.0.110/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Pulled - Container image "quay.io/rhoai/odh-feast-operator-rhel9@sha256:4245ccd54ad041195c289e140b81a7407d8a4be0e63feec3da83f015cabf1bd7" already present on machine
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: AddedInterface - Add eth0 [10.133.0.111/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Created - Created container: manager
2026/09/07 21:37:31   21:34:34 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-8vkv9: Started - Started container manager
2026/09/07 21:37:31   21:34:34 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-5fc6bb8d67 from 0 to 1
2026/09/07 21:37:31   21:34:34 INFO ReplicaSet/feast-operator-controller-manager-5fc6bb8d67: SuccessfulCreate - Created pod: feast-operator-controller-manager-5fc6bb8d67-8vkv9
2026/09/07 21:37:31   21:34:33 INFO Pod/maas-controller-56c898799-vcwmz: Scheduled - Successfully assigned redhat-ods-applications/maas-controller-56c898799-vcwmz to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:33 INFO Deployment/maas-controller: ScalingReplicaSet - Scaled up replica set maas-controller-56c898799 from 0 to 1
2026/09/07 21:37:31   21:34:33 INFO ReplicaSet/maas-controller-56c898799: SuccessfulCreate - Created pod: maas-controller-56c898799-vcwmz
2026/09/07 21:37:31   21:34:33 WARN Pod/maas-controller-56c898799-vcwmz: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "maas-controller-webhook-cert" not found
2026/09/07 21:37:31   21:34:32 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:29 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:29 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:28 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:28 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:28 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Started - Started container webhook
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Created - Created container: controller
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Started - Started container controller
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Pulling - Pulling image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2"
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-k8s-operator-rhel9@sha256:b4b73d6b9ca695dc8d80756f64d84c865de484d12e315dbf731413cc831977b2" in 583ms (583ms including waiting). Image size: 193888070 bytes.
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Created - Created container: manager
2026/09/07 21:37:31   21:34:28 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Started - Started container manager
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Created - Created container: webhook
2026/09/07 21:37:31   21:34:28 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:28 INFO Pod/spark-operator-controller-c594b967f-m4pdj: AddedInterface - Add eth0 [10.133.0.109/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:27 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: Scheduled - Successfully assigned redhat-ods-applications/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:27 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-webhook-5d466b7d66-nknqx to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-controller-c594b967f-m4pdj: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-controller-c594b967f-m4pdj to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: AddedInterface - Add eth0 [10.134.0.95/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:27 INFO Pod/spark-operator-webhook-5d466b7d66-nknqx: Pulled - Container image "quay.io/rhoai/odh-spark-operator-rhel9@sha256:a731f346c91ca43802543e004cb8b3c6ed0498818773dac41cd1a2ce447aed0c" already present on machine
2026/09/07 21:37:31   21:34:27 INFO Deployment/spark-operator-controller: ScalingReplicaSet - Scaled up replica set spark-operator-controller-c594b967f from 0 to 1
2026/09/07 21:37:31   21:34:27 INFO ReplicaSet/spark-operator-controller-c594b967f: SuccessfulCreate - Created pod: spark-operator-controller-c594b967f-m4pdj
2026/09/07 21:37:31   21:34:27 INFO PodDisruptionBudget/ogx-k8s-operator-controller-manager-pdb: NoPods - No matching pods found
2026/09/07 21:37:31   21:34:27 INFO ReplicaSet/ogx-k8s-operator-controller-manager-59777cdbbb: SuccessfulCreate - Created pod: ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8
2026/09/07 21:37:31   21:34:27 INFO Deployment/spark-operator-webhook: ScalingReplicaSet - Scaled up replica set spark-operator-webhook-5d466b7d66 from 0 to 1
2026/09/07 21:37:31   21:34:27 INFO Deployment/ogx-k8s-operator-controller-manager: ScalingReplicaSet - Scaled up replica set ogx-k8s-operator-controller-manager-59777cdbbb from 0 to 1
2026/09/07 21:37:31   21:34:27 INFO ReplicaSet/spark-operator-webhook-5d466b7d66: SuccessfulCreate - Created pod: spark-operator-webhook-5d466b7d66-nknqx
2026/09/07 21:37:31   21:34:27 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:27 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-qc6d8: AddedInterface - Add eth0 [10.132.0.93/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:26 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:24 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:24 INFO Lease/ai-gateway-lock: LeaderElection - ai-gateway-operator-89f69845b-575wm_5f3bb8f8-c55a-4938-87bf-61beb6f80742 became leader
2026/09/07 21:37:31   21:34:24 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Killing - Stopping container manager
2026/09/07 21:37:31   21:34:24 INFO Lease/kserve-module-controller-manager: LeaderElection - kserve-module-controller-manager-66ff7c977-mggzt_7922e366-ed21-468f-ba55-50b96b9aa153 became leader
2026/09/07 21:37:31   21:34:24 INFO Lease/opendatahub-feast-operator-lock: LeaderElection - opendatahub-feast-operator-84ff599545-d49rd_527382f3-cc39-422e-b551-f2e4845a1781 became leader
2026/09/07 21:37:31   21:34:21 INFO Lease/opendatahub-ogx-operator-lock: LeaderElection - opendatahub-ogx-operator-644d45d5b-kvfc5_12f3105b-24c5-421f-ab29-b88a77d95e51 became leader
2026/09/07 21:37:31   21:34:20 INFO Lease/trainer.kubeflow.org: LeaderElection - kubeflow-trainer-controller-manager-5449979c85-m4cpw_dc9aa37f-4b29-4d8f-8ab2-5ed6f7209db0 became leader
2026/09/07 21:37:31   21:34:20 INFO Lease/spark-operator-module-controller-manager: LeaderElection - spark-operator-module-controller-manager-59549b5854-gnzt9_f3468754-399c-4ba9-ae79-cd46fa47a561 became leader
2026/09/07 21:37:31   21:34:15 WARN Pod/rhods-dashboard-77f45899f6-84xgj: Unhealthy - Readiness probe failed: Get "https://10.132.0.59:8444/healthz": dial tcp 10.132.0.59:8444: connect: connection refused
2026/09/07 21:37:31   21:34:15 WARN Pod/rhods-dashboard-77f45899f6-84xgj: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:34:15 WARN Pod/rhods-dashboard-77f45899f6-84xgj: Unhealthy - Readiness probe failed: Get "https://10.132.0.59:8943/healthcheck": dial tcp 10.132.0.59:8943: connect: connection refused
2026/09/07 21:37:31   21:34:12 WARN Pod/rhods-dashboard-77f45899f6-zxcz4: Unhealthy - Readiness probe failed: Get "https://10.133.0.50:8444/healthz": dial tcp 10.133.0.50:8444: connect: connection refused
2026/09/07 21:37:31   21:34:10 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728" in 2.514s (2.514s including waiting). Image size: 186574763 bytes.
2026/09/07 21:37:31   21:34:10 INFO Lease/bed7462b.x-k8s.io: LeaderElection - mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf_362bff1f-57cf-4710-8a24-8e4738bb6ebd became leader
2026/09/07 21:37:31   21:34:10 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Created - Created container: manager
2026/09/07 21:37:31   21:34:10 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Started - Started container manager
2026/09/07 21:37:31   21:34:07 WARN Pod/rhods-dashboard-77f45899f6-zxcz4: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:37:31   21:34:07 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: AddedInterface - Add eth0 [10.132.0.92/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:07 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-operator-rhel9@sha256:b9e809f5b7665ba7682857f0a365feafe06e107f7d9cb58172b91b2257696728"
2026/09/07 21:37:31   21:34:07 WARN Pod/rhods-dashboard-77f45899f6-zxcz4: Unhealthy - Readiness probe failed: Get "https://10.133.0.50:8943/healthcheck": dial tcp 10.133.0.50:8943: connect: connection refused
2026/09/07 21:37:31   21:34:06 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:06 WARN ServiceMonitor/mcp-lifecycle-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/mcp-lifecycle-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:06 INFO Pod/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:06 INFO Pod/ai-gateway-operator-89f69845b-575wm: Started - Started container manager
2026/09/07 21:37:31   21:34:06 INFO ReplicaSet/mcp-lifecycle-operator-controller-manager-5f798b84bf: SuccessfulCreate - Created pod: mcp-lifecycle-operator-controller-manager-5f798b84bf-lwwdf
2026/09/07 21:37:31   21:34:06 INFO Deployment/mcp-lifecycle-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-operator-controller-manager-5f798b84bf from 0 to 1
2026/09/07 21:37:31   21:34:06 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Pulled - Successfully pulled image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810" in 1.506s (1.506s including waiting). Image size: 173568461 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Started - Started container manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 284ms (284ms including waiting). Image size: 194988415 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Pulled - Successfully pulled image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845" in 425ms (425ms including waiting). Image size: 168389130 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Pulling - Pulling image "quay.io/rhoai/odh-kf-notebook-controller-rhel9@sha256:6ef921ba366c8b828160bb8c5722e58701944fd9b3213a897bbea4774d443845"
2026/09/07 21:37:31   21:34:05 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: AddedInterface - Add eth0 [10.134.0.94/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:05 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 21:37:31   21:34:05 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 378ms (378ms including waiting). Image size: 207018016 bytes.
2026/09/07 21:37:31   21:34:05 INFO Pod/ai-gateway-operator-89f69845b-575wm: Created - Created container: manager
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 21:37:31   21:34:05 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 388ms (389ms including waiting). Image size: 209286205 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Created - Created container: copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulled - Successfully pulled image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3" in 3.577s (3.577s including waiting). Image size: 194988415 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Started - Started container manager
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Started - Started container copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b" in 3.629s (3.629s including waiting). Image size: 227888997 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Created - Created container: copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/ai-gateway-operator-89f69845b-575wm: Started - Started container copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/ai-gateway-operator-89f69845b-575wm: Created - Created container: copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulled - Successfully pulled image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567" in 3.596s (3.596s including waiting). Image size: 209286205 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Created - Created container: manager
2026/09/07 21:37:31   21:34:04 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Started - Started container copy-manifests
2026/09/07 21:37:31   21:34:04 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulled - Successfully pulled image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189" in 3.409s (3.409s including waiting). Image size: 207018016 bytes.
2026/09/07 21:37:31   21:34:04 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Started - Started container manager
2026/09/07 21:37:31   21:34:04 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: AddedInterface - Add eth0 [10.133.0.108/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:04 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Pulling - Pulling image "quay.io/rhoai/odh-notebook-controller-rhel9@sha256:066f9b4441ef44426fe3536d3634fe4c8caa5353894a423801ca3bfe25208810"
2026/09/07 21:37:31   21:34:04 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Created - Created container: manager
2026/09/07 21:37:31   21:34:03 INFO Pod/odh-notebook-controller-manager-59b75488c-h96tw: Scheduled - Successfully assigned redhat-ods-applications/odh-notebook-controller-manager-59b75488c-h96tw to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Created - Created container: manager
2026/09/07 21:37:31   21:34:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Pulled - Successfully pulled image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35" in 1.501s (1.501s including waiting). Image size: 158342259 bytes.
2026/09/07 21:37:31   21:34:03 WARN Pod/odh-notebook-controller-manager-59b75488c-h96tw: FailedMount - MountVolume.SetUp failed for volume "webhook-cert" : secret "odh-notebook-controller-webhook-cert" not found
2026/09/07 21:37:31   21:34:03 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Pulled - Successfully pulled image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637" in 2.693s (2.693s including waiting). Image size: 191954597 bytes.
2026/09/07 21:37:31   21:34:03 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Started - Started container manager
2026/09/07 21:37:31   21:34:03 INFO ReplicaSet/odh-notebook-controller-manager-59b75488c: SuccessfulCreate - Created pod: odh-notebook-controller-manager-59b75488c-h96tw
2026/09/07 21:37:31   21:34:02 INFO Pod/notebook-controller-deployment-75c8689979-hnkpd: Scheduled - Successfully assigned redhat-ods-applications/notebook-controller-deployment-75c8689979-hnkpd to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:02 INFO Deployment/notebook-controller-deployment: ScalingReplicaSet - Scaled up replica set notebook-controller-deployment-75c8689979 from 0 to 1
2026/09/07 21:37:31   21:34:02 INFO Deployment/odh-notebook-controller-manager: ScalingReplicaSet - Scaled up replica set odh-notebook-controller-manager-59b75488c from 0 to 1
2026/09/07 21:37:31   21:34:02 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: AddedInterface - Add eth0 [10.133.0.107/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:02 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Pulling - Pulling image "quay.io/rhoai/odh-trainer-rhel9@sha256:a5fb7ccc228478c317ed3cb8beac2b95c0b3f9a366d4077a36a5e17599776c35"
2026/09/07 21:37:31   21:34:02 INFO ReplicaSet/notebook-controller-deployment-75c8689979: SuccessfulCreate - Created pod: notebook-controller-deployment-75c8689979-hnkpd
2026/09/07 21:37:31   21:34:01 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:34:01 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-m4cpw: Scheduled - Successfully assigned redhat-ods-applications/kubeflow-trainer-controller-manager-5449979c85-m4cpw to ip-10-0-140-136.ec2.internal
2026/09/07 21:37:31   21:34:01 INFO Pod/ai-gateway-operator-89f69845b-575wm: Pulling - Pulling image "quay.io/rhoai/odh-ai-gateway-operator-rhel9@sha256:21f9656320f0e343329624b89b22967ee4d7a218ee8de08a094234bfacc8c189"
2026/09/07 21:37:31   21:34:01 INFO Deployment/kubeflow-trainer-controller-manager: ScalingReplicaSet - Scaled up replica set kubeflow-trainer-controller-manager-5449979c85 from 0 to 1
2026/09/07 21:37:31   21:34:01 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: AddedInterface - Add eth0 [10.132.0.91/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Pulling - Pulling image "quay.io/rhoai/odh-spark-operator-module-rhel9@sha256:82a8b702af8e535d8e6b01f563043d8926aacf7ef277cd3ca6bb9875430c9637"
2026/09/07 21:37:31   21:34:01 INFO Pod/ai-gateway-operator-89f69845b-575wm: AddedInterface - Add eth0 [10.134.0.90/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Pulling - Pulling image "quay.io/rhoai/odh-kserve-module-operator-rhel9@sha256:db87915368dd6c91975767be2055a068aeaf899603af491199e433f8a65b795b"
2026/09/07 21:37:31   21:34:01 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: AddedInterface - Add eth0 [10.134.0.91/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Pulling - Pulling image "quay.io/rhoai/odh-feast-module-operator-rhel9@sha256:01b735b621462181207769bde7191e1423d8351ae7ff971c251b8a4ff20ec567"
2026/09/07 21:37:31   21:34:01 INFO ReplicaSet/kubeflow-trainer-controller-manager-5449979c85: SuccessfulCreate - Created pod: kubeflow-trainer-controller-manager-5449979c85-m4cpw
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: AddedInterface - Add eth0 [10.134.0.92/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: AddedInterface - Add eth0 [10.134.0.93/23] from ovn-kubernetes
2026/09/07 21:37:31   21:34:01 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Pulling - Pulling image "quay.io/rhoai/odh-ogx-module-operator-rhel9@sha256:b9fcb0aa7f43e7184a1143e8f152e0a7b2b4a00ad89effc878c3a4c01a39a8b3"
2026/09/07 21:37:31   21:34:00 INFO Pod/spark-operator-module-controller-manager-59549b5854-gnzt9: Scheduled - Successfully assigned redhat-ods-applications/spark-operator-module-controller-manager-59549b5854-gnzt9 to ip-10-0-137-227.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/opendatahub-ogx-operator-644d45d5b-kvfc5: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-ogx-operator-644d45d5b-kvfc5 to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/opendatahub-feast-operator-84ff599545-d49rd: Scheduled - Successfully assigned redhat-ods-applications/opendatahub-feast-operator-84ff599545-d49rd to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Scheduled - Successfully assigned redhat-ods-applications/kserve-module-controller-manager-66ff7c977-mggzt to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO Pod/ai-gateway-operator-89f69845b-575wm: Scheduled - Successfully assigned redhat-ods-applications/ai-gateway-operator-89f69845b-575wm to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/spark-operator-module-controller-manager-59549b5854: SuccessfulCreate - Created pod: spark-operator-module-controller-manager-59549b5854-gnzt9
2026/09/07 21:37:31   21:34:00 INFO Deployment/kserve-module-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-module-controller-manager-66ff7c977 from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO Deployment/spark-operator-module-controller-manager: ScalingReplicaSet - Scaled up replica set spark-operator-module-controller-manager-59549b5854 from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/ai-gateway-operator-89f69845b: SuccessfulCreate - Created pod: ai-gateway-operator-89f69845b-575wm
2026/09/07 21:37:31   21:34:00 INFO Deployment/ai-gateway-operator: ScalingReplicaSet - Scaled up replica set ai-gateway-operator-89f69845b from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/kserve-module-controller-manager-66ff7c977: SuccessfulCreate - Created pod: kserve-module-controller-manager-66ff7c977-mggzt
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/opendatahub-feast-operator-84ff599545: SuccessfulCreate - Created pod: opendatahub-feast-operator-84ff599545-d49rd
2026/09/07 21:37:31   21:34:00 INFO Deployment/opendatahub-ogx-operator: ScalingReplicaSet - Scaled up replica set opendatahub-ogx-operator-644d45d5b from 0 to 1
2026/09/07 21:37:31   21:34:00 INFO ReplicaSet/opendatahub-ogx-operator-644d45d5b: SuccessfulCreate - Created pod: opendatahub-ogx-operator-644d45d5b-kvfc5
2026/09/07 21:37:31   21:34:00 INFO Deployment/opendatahub-feast-operator: ScalingReplicaSet - Scaled up replica set opendatahub-feast-operator-84ff599545 from 0 to 1
2026/09/07 21:37:31   21:33:59 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Started - Started container manager
2026/09/07 21:37:31   21:33:59 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Created - Created container: manager
2026/09/07 21:37:31   21:33:59 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Started - Started container manager
2026/09/07 21:37:31   21:33:59 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Pulled - Successfully pulled image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4" in 2.334s (2.334s including waiting). Image size: 143076452 bytes.
2026/09/07 21:37:31   21:33:59 INFO Lease/mcp-lifecycle-module-operator-leader: LeaderElection - mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r_69b6915b-f8dd-475a-b53b-97b65053fd4f became leader
2026/09/07 21:37:31   21:33:59 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Created - Created container: manager
2026/09/07 21:37:31   21:33:58 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Pulled - Container image "quay.io/rhoai/odh-workbenches-operator-rhel9@sha256:bddb6bb5c754ee7d077ddea53dae4345193b886cd0208f320f2b1ce051934898" already present on machine
2026/09/07 21:37:31   21:33:56 INFO Pod/workbenches-operator-c674d9c9-xc2w6: Scheduled - Successfully assigned redhat-ods-applications/workbenches-operator-c674d9c9-xc2w6 to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Scheduled - Successfully assigned redhat-ods-applications/trainer-operator-controller-manager-6b585cbb9f-5vjcn to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:56 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Scheduled - Successfully assigned redhat-ods-applications/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Started - Started container manager
2026/09/07 21:37:31   21:33:56 INFO ReplicaSet/mcp-lifecycle-module-operator-controller-manager-6b668bbbb: SuccessfulCreate - Created pod: mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r
2026/09/07 21:37:31   21:33:56 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: Pulling - Pulling image "quay.io/rhoai/odh-mcp-lifecycle-module-operator-rhel9@sha256:6f275c294e89ef3613de8a1e824c815cb9876b1b4123f354df86092bc06402f4"
2026/09/07 21:37:31   21:33:56 INFO Pod/mcp-lifecycle-module-operator-controller-manager-6b668bbbbff46r: AddedInterface - Add eth0 [10.134.0.87/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: AddedInterface - Add eth0 [10.134.0.88/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Pulled - Container image "quay.io/rhoai/odh-trainer-operator-rhel9@sha256:82d34a6f9644f5d910ad41e473b8c83e1ee54076368f3e7ba48ed6234b1df0e2" already present on machine
2026/09/07 21:37:31   21:33:56 INFO Pod/workbenches-operator-c674d9c9-xc2w6: AddedInterface - Add eth0 [10.134.0.89/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:56 INFO Deployment/mcp-lifecycle-module-operator-controller-manager: ScalingReplicaSet - Scaled up replica set mcp-lifecycle-module-operator-controller-manager-6b668bbbb from 0 to 1
2026/09/07 21:37:31   21:33:56 INFO Deployment/trainer-operator-controller-manager: ScalingReplicaSet - Scaled up replica set trainer-operator-controller-manager-6b585cbb9f from 0 to 1
2026/09/07 21:37:31   21:33:56 INFO ReplicaSet/trainer-operator-controller-manager-6b585cbb9f: SuccessfulCreate - Created pod: trainer-operator-controller-manager-6b585cbb9f-5vjcn
2026/09/07 21:37:31   21:33:56 INFO ReplicaSet/workbenches-operator-c674d9c9: SuccessfulCreate - Created pod: workbenches-operator-c674d9c9-xc2w6
2026/09/07 21:37:31   21:33:56 INFO Deployment/workbenches-operator: ScalingReplicaSet - Scaled up replica set workbenches-operator-c674d9c9 from 0 to 1
2026/09/07 21:37:31   21:33:56 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-5vjcn: Created - Created container: manager
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: AddedInterface - Add eth0 [10.134.0.86/23] from ovn-kubernetes
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Created - Created container: manager
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Pulled - Successfully pulled image "registry.k8s.io/pause:3.9" in 338ms (338ms including waiting). Image size: 750414 bytes.
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Pulling - Pulling image "registry.k8s.io/pause:3.9"
2026/09/07 21:37:31   21:33:54 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Started - Started container manager
2026/09/07 21:37:31   21:33:53 INFO Pod/feast-operator-controller-manager-6b8fb45c8b-lrgkd: Scheduled - Successfully assigned redhat-ods-applications/feast-operator-controller-manager-6b8fb45c8b-lrgkd to ip-10-0-130-155.ec2.internal
2026/09/07 21:37:31   21:33:53 INFO ReplicaSet/feast-operator-controller-manager-6b8fb45c8b: SuccessfulCreate - Created pod: feast-operator-controller-manager-6b8fb45c8b-lrgkd
2026/09/07 21:37:31   21:33:53 INFO Deployment/feast-operator-controller-manager: ScalingReplicaSet - Scaled up replica set feast-operator-controller-manager-6b8fb45c8b from 0 to 1
2026/09/07 21:37:31   21:33:51 INFO Pod/dashboard-operator-669d6564bd-zmv4v: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:50 INFO Pod/trainer-operator-controller-manager-6b585cbb9f-f76bn: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/ai-gateway-operator-89f69845b-vrn7g: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/workbenches-operator-c674d9c9-j56wt: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/odh-observability-59db79cb46-fklv9: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/aihub-controller-manager-d877888b8-kdvt2: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:50 INFO Pod/kserve-module-controller-manager-66ff7c977-8qg2c: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:49 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:49 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:49 INFO Pod/opendatahub-feast-operator-84ff599545-q9xzw: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:49 INFO Pod/mlflow-operator-controller-manager-5bbb468f5f-4lrz5: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:48 INFO Pod/spark-operator-module-controller-manager-59549b5854-kxzpn: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:48 INFO Pod/opendatahub-ogx-operator-644d45d5b-ngf9c: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/mlflow-ui-7bb6d46f5f-dh94z: Killing - Stopping container mlflow-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/catalog-controller-manager-55967bdd4b-8psf9: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/maas-ui-68f944f6b6-ph28z: Killing - Stopping container maas-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/kubeflow-trainer-controller-manager-5449979c85-kq4s7: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/model-registry-operator-controller-manager-59c8bf9b89-4b76q: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/dashboard-redirect-79754c669f-6vzlx: Killing - Stopping container nginx
2026/09/07 21:37:31   21:33:46 INFO Pod/odh-notebook-controller-manager-5f845bd4cb-6fz56: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/eval-hub-ui-869b89d9b4-glpj6: Killing - Stopping container eval-hub-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/odh-model-controller-6b9dd8b59d-2dggj: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/model-registry-ui-7c777bcbb7-cfggf: Killing - Stopping container model-registry-ui
2026/09/07 21:37:31   21:33:46 WARN Pod/llmisvc-controller-manager-6df7b6c5d4-k658k: Unhealthy - Readiness probe failed: Get "http://10.134.0.52:8081/readyz": dial tcp 10.134.0.52:8081: connect: connection refused
2026/09/07 21:37:31   21:33:46 INFO Pod/notebooks-ui-6577cc467-5gsgg: Killing - Stopping container notebooks-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/dashboard-redirect-79754c669f-skpdj: Killing - Stopping container nginx
2026/09/07 21:37:31   21:33:46 INFO Pod/automl-ui-74d9b97d49-dl67k: Killing - Stopping container automl-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/gen-ai-ui-c58764dbf-j8tl2: Killing - Stopping container gen-ai-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-84xgj: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:33:46 INFO Pod/notebook-controller-deployment-75c8689979-4zdpb: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:46 INFO Pod/agent-ops-ui-57c49dcf65-2qgld: Killing - Stopping container agent-ops-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/autorag-ui-58f548cbd-df2sv: Killing - Stopping container autorag-ui
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-84xgj: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-zxcz4: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-zxcz4: Killing - Stopping container core-bff
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-zxcz4: Killing - Stopping container rhods-dashboard
2026/09/07 21:37:31   21:33:46 INFO Pod/rhods-dashboard-77f45899f6-84xgj: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:37:31   21:33:45 INFO Pod/data-science-pipelines-operator-controller-manager-56b8f892mnvm: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/ogx-k8s-operator-controller-manager-59777cdbbb-5mwrb: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k658k: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/spark-operator-webhook-5d466b7d66-tqjkr: Killing - Stopping container webhook
2026/09/07 21:37:31   21:33:45 INFO Pod/spark-operator-controller-c594b967f-vxjlm: Killing - Stopping container controller
2026/09/07 21:37:31   21:33:45 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-fsgxh: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/maas-controller-56c898799-tm9z4: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/kuberay-operator-7d87f54798-nnvm6: Killing - Stopping container kuberay-operator
2026/09/07 21:37:31   21:33:45 INFO Pod/kserve-controller-manager-5fbbd6dd88-rtwp8: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:45 INFO Pod/model-serving-api-7df8c9655f-5lmjx: Killing - Stopping container server
2026/09/07 21:37:31   21:33:45 INFO Pod/feast-operator-controller-manager-5fc6bb8d67-g8m9q: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:44 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:44 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:44 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:43 INFO Pod/trustyai-service-operator-controller-manager-d6bf47757-lj2gq: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/aihub-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/aihub-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 WARN ServiceMonitor/model-registry-operator-controller-manager-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/model-registry-operator-controller-manager-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:37:31   21:33:42 INFO Pod/kserve-localmodelnode-agent-t2kzl: Killing - Stopping container manager
2026/09/07 21:37:31   21:33:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-nf6kh
2026/09/07 21:37:31   21:33:42 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-t2kzl
2026/09/07 21:37:31   21:33:42 INFO Pod/kserve-localmodelnode-agent-nf6kh: Killing - Stopping container manager
2026/09/07 21:37:31 === RESOURCE QUOTAS ===
2026/09/07 21:37:31 No resource quotas found in monitored namespaces
2026/09/07 21:37:31 Classification for Validate resource deletion recovery: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/07 21:37:31   Evidence: event Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get "https://10.133.0.117:8444/healthz": dial tcp 10.133.0.117:8444: connect: connection refused
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get \"https://10.133.0.117:8444/healthz\": dial tcp 10.133.0.117:8444: connect: connection refused"],"confidence":"medium"}
2026/09/07 21:37:31 Diagnostics complete for failed test: Validate resource deletion recovery
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/07 21:40:57 TEST FAILURE DETECTED: group 1
2026/09/07 21:40:57 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 1) ===
2026/09/07 21:40:57 === CLUSTER STATE ===
2026/09/07 21:40:57 Node: ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:40:57   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:40:57   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:40:57   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:40:57   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:40:57 Node: ip-10-0-137-227.ec2.internal
2026/09/07 21:40:57   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:40:57   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:40:57   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:40:57   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:40:57   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:40:57 Node: ip-10-0-140-136.ec2.internal
2026/09/07 21:40:57   Condition MemoryPressure: False - kubelet has sufficient memory available
2026/09/07 21:40:57   Condition DiskPressure: False - kubelet has no disk pressure
2026/09/07 21:40:57   Condition PIDPressure: False - kubelet has sufficient PID available
2026/09/07 21:40:57   Condition Ready: True - kubelet is posting ready status
2026/09/07 21:40:57   Resources: 7500m CPU, 31236216Ki memory allocatable (of 8000m CPU, 32387192Ki memory capacity)
2026/09/07 21:40:57 === DEPLOYMENTS ===
2026/09/07 21:40:57 Namespace: redhat-ods-applications
2026/09/07 21:40:57   Deployment agent-ops-ui: 1/1 (READY)
2026/09/07 21:40:57   Deployment dashboard-operator: 2/2 (READY)
2026/09/07 21:40:57   Deployment dashboard-redirect: 2/2 (READY)
2026/09/07 21:40:57   Deployment gen-ai-ui: 1/1 (READY)
2026/09/07 21:40:57   Deployment maas-ui: 1/1 (READY)
2026/09/07 21:40:57   Deployment notebooks-ui: 1/1 (READY)
2026/09/07 21:40:57   Deployment rhods-dashboard: 2/2 (READY)
2026/09/07 21:40:57 Namespace: kube-system
2026/09/07 21:40:57 === PODS ===
2026/09/07 21:40:57 Namespace redhat-ods-applications: no problematic pods found
2026/09/07 21:40:57 Namespace kube-system: no problematic pods found
2026/09/07 21:40:57 === OPERATOR STATUS ===
2026/09/07 21:40:57 Operator deployment: 3/3 ready
2026/09/07 21:40:57 === DSCI STATUS ===
2026/09/07 21:40:57 Failed to collect DSCI data: default-dsci unhealthy conditions: MonitoringReady=False; MonitoringStackAvailable=False; ThanosQuerierAvailable=False; TempoAvailable=False; OpenTelemetryCollectorAvailable=False; AlertingAvailable=False; NodeMetricsEndpointAvailable=False; PersesAvailable=False
2026/09/07 21:40:57 === DSC STATUS ===
2026/09/07 21:40:57 DSC default-dsc:
2026/09/07 21:40:57   Ready: True - 
2026/09/07 21:40:57   ProvisioningSucceeded: True - 
2026/09/07 21:40:57   ComponentsReady: True - All registered components have ManagementState Removed or are not configured
2026/09/07 21:40:57   ModulesReady: True - 
2026/09/07 21:40:57   AIGatewayReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   AIPipelinesReady: False - Component ManagementState is set to Removed
2026/09/07 21:40:57   BatchGatewayReady: False - Submodule ManagementState is set to Removed
2026/09/07 21:40:57   DashboardReady: True - 
2026/09/07 21:40:57   FeastOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   KserveLLMInferenceServiceDependencies: False - Submodule ManagementState is set to Removed
2026/09/07 21:40:57   KserveLLMInferenceServiceWideEPDependencies: False - Submodule ManagementState is set to Removed
2026/09/07 21:40:57   KserveReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   KueueReady: False - Component ManagementState is set to Removed
2026/09/07 21:40:57   MCPLifecycleOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   MLflowOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   ModelRegistryReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   ModelsAsAServiceReady: False - Submodule ManagementState is set to Removed
2026/09/07 21:40:57   OGXReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   RayReady: False - Component ManagementState is set to Removed
2026/09/07 21:40:57   SparkOperatorReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   TrainerReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   TrustyAIReady: False - Component ManagementState is set to Removed
2026/09/07 21:40:57   WorkbenchesReady: False - Module ManagementState is set to Removed
2026/09/07 21:40:57   WorkbenchesV2Ready: False - Submodule ManagementState is set to Removed
2026/09/07 21:40:57 === RECENT EVENTS ===
2026/09/07 21:40:57   21:40:50 INFO Pod/kserve-module-controller-manager-66ff7c977-mggzt: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:48 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:40:48 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:40:48 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:40:48 INFO Pod/kserve-localmodelnode-agent-8dkgk: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:48 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:48 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-8dkgk
2026/09/07 21:40:57   21:40:48 INFO Pod/odh-model-controller-57457c4c89-h2ctc: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:48 INFO Pod/model-serving-api-7df8c9655f-jv8p2: Killing - Stopping container server
2026/09/07 21:40:57   21:40:48 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:48 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:47 INFO DaemonSet/kserve-localmodelnode-agent: SuccessfulDelete - Deleted pod: kserve-localmodelnode-agent-95jw9
2026/09/07 21:40:57   21:40:47 INFO Pod/kserve-localmodelnode-agent-95jw9: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:40 INFO Lease/odh-model-controller.opendatahub.io: LeaderElection - odh-model-controller-57457c4c89-h2ctc_50a9cd2b-4aea-4e15-b029-e2c1e2d42f1d became leader
2026/09/07 21:40:57   21:40:25 INFO Pod/odh-model-controller-57457c4c89-h2ctc: Started - Started container manager
2026/09/07 21:40:57   21:40:25 INFO Pod/odh-model-controller-57457c4c89-h2ctc: Created - Created container: manager
2026/09/07 21:40:57   21:40:24 INFO Pod/odh-model-controller-57457c4c89-h2ctc: AddedInterface - Add eth0 [10.133.0.121/23] from ovn-kubernetes
2026/09/07 21:40:57   21:40:24 INFO Pod/odh-model-controller-57457c4c89-h2ctc: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 21:40:57   21:40:24 INFO Pod/odh-model-controller-57457c4c89-h2ctc: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 391ms (391ms including waiting). Image size: 239132625 bytes.
2026/09/07 21:40:57   21:40:23 INFO Pod/odh-model-controller-57457c4c89-h2ctc: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-h2ctc to ip-10-0-140-136.ec2.internal
2026/09/07 21:40:57   21:40:23 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-h2ctc
2026/09/07 21:40:57   21:40:23 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 0 to 1
2026/09/07 21:40:57   21:40:21 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:21 INFO Pod/odh-model-controller-57457c4c89-nmnhd: Killing - Stopping container manager
2026/09/07 21:40:57   21:40:09 INFO Pod/model-serving-api-7df8c9655f-jv8p2: Created - Created container: server
2026/09/07 21:40:57   21:40:09 INFO Pod/model-serving-api-7df8c9655f-jv8p2: AddedInterface - Add eth0 [10.133.0.120/23] from ovn-kubernetes
2026/09/07 21:40:57   21:40:09 INFO Pod/model-serving-api-7df8c9655f-jv8p2: Started - Started container server
2026/09/07 21:40:57   21:40:09 INFO Pod/model-serving-api-7df8c9655f-jv8p2: Pulled - Container image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" already present on machine
2026/09/07 21:40:57   21:40:08 INFO Pod/model-serving-api-7df8c9655f-jv8p2: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-jv8p2 to ip-10-0-140-136.ec2.internal
2026/09/07 21:40:57   21:40:08 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 0 to 1
2026/09/07 21:40:57   21:40:08 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-jv8p2
2026/09/07 21:40:57   21:40:06 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Killing - Stopping container server
2026/09/07 21:40:57   21:40:06 INFO Pod/model-serving-api-7df8c9655f-k82t5: Killing - Stopping container server
2026/09/07 21:40:57   21:39:51 INFO Lease/llminferenceservice-kserve-controller-manager: LeaderElection - llmisvc-controller-manager-6df7b6c5d4-chf6g_2d749a45-3974-4faa-aff7-0c7e4bfab37e became leader
2026/09/07 21:40:57   21:39:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 326ms (326ms including waiting). Image size: 235279660 bytes.
2026/09/07 21:40:57   21:39:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: Started - Started container manager
2026/09/07 21:40:57   21:39:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 21:40:57   21:39:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: AddedInterface - Add eth0 [10.134.0.116/23] from ovn-kubernetes
2026/09/07 21:40:57   21:39:33 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: Created - Created container: manager
2026/09/07 21:40:57   21:39:32 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-chf6g: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-chf6g to ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   21:39:32 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-chf6g
2026/09/07 21:40:57   21:39:32 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 0 to 1
2026/09/07 21:40:57   21:39:31 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Killing - Stopping container manager
2026/09/07 21:40:57   21:39:31 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-k7j5f: Killing - Stopping container manager
2026/09/07 21:40:57   21:38:59 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: Created - Created container: manager
2026/09/07 21:40:57   21:38:59 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8" in 480ms (480ms including waiting). Image size: 244249302 bytes.
2026/09/07 21:40:57   21:38:59 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: Started - Started container manager
2026/09/07 21:40:57   21:38:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: Scheduled - Successfully assigned redhat-ods-applications/kserve-localmodel-controller-manager-fc8c879d7-l2j9m to ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   21:38:58 INFO Deployment/kserve-localmodel-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-localmodel-controller-manager-fc8c879d7 from 0 to 1
2026/09/07 21:40:57   21:38:58 INFO ReplicaSet/kserve-localmodel-controller-manager-fc8c879d7: SuccessfulCreate - Created pod: kserve-localmodel-controller-manager-fc8c879d7-l2j9m
2026/09/07 21:40:57   21:38:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: Pulling - Pulling image "quay.io/rhoai/odh-kserve-localmodel-controller-rhel9@sha256:01a0250dfbd5a2abd3d55ba5adf6ab0b47f2ae46c085c06b7e2dc8f92aa6e5e8"
2026/09/07 21:40:57   21:38:58 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-l2j9m: AddedInterface - Add eth0 [10.134.0.115/23] from ovn-kubernetes
2026/09/07 21:40:57   21:38:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-4sgft: Killing - Stopping container manager
2026/09/07 21:40:57   21:38:56 INFO Pod/kserve-localmodel-controller-manager-fc8c879d7-lg2tv: Killing - Stopping container manager
2026/09/07 21:40:57   21:38:40 INFO Lease/kserve-controller-manager-leader-lock: LeaderElection - kserve-controller-manager-5fbbd6dd88-88vb9_7dd3a6d0-67ca-49b9-be95-cf24cd3ee2be became leader
2026/09/07 21:40:57   21:38:24 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: Started - Started container manager
2026/09/07 21:40:57   21:38:24 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: AddedInterface - Add eth0 [10.134.0.114/23] from ovn-kubernetes
2026/09/07 21:40:57   21:38:24 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: Pulling - Pulling image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3"
2026/09/07 21:40:57   21:38:24 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-controller-rhel9@sha256:0efe283dfad7e83e712a64c86a266a65fd4ebe9aadc37e02c03c324c762011a3" in 319ms (319ms including waiting). Image size: 247667445 bytes.
2026/09/07 21:40:57   21:38:24 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: Created - Created container: manager
2026/09/07 21:40:57   21:38:23 INFO Pod/kserve-controller-manager-5fbbd6dd88-88vb9: Scheduled - Successfully assigned redhat-ods-applications/kserve-controller-manager-5fbbd6dd88-88vb9 to ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   21:38:23 INFO Deployment/kserve-controller-manager: ScalingReplicaSet - Scaled up replica set kserve-controller-manager-5fbbd6dd88 from 0 to 1
2026/09/07 21:40:57   21:38:23 INFO ReplicaSet/kserve-controller-manager-5fbbd6dd88: SuccessfulCreate - Created pod: kserve-controller-manager-5fbbd6dd88-88vb9
2026/09/07 21:40:57   21:38:21 INFO Pod/kserve-controller-manager-5fbbd6dd88-n8xx4: Killing - Stopping container manager
2026/09/07 21:40:57   21:38:21 INFO Pod/kserve-controller-manager-5fbbd6dd88-ph2cn: Killing - Stopping container manager
2026/09/07 21:40:57   21:37:44 WARN Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:40:57   21:37:44 WARN Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get "https://10.133.0.117:8943/healthcheck": dial tcp 10.133.0.117:8943: connect: connection refused
2026/09/07 21:40:57   21:37:44 WARN Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe failed: Get "https://10.133.0.117:8444/healthz": dial tcp 10.133.0.117:8444: connect: connection refused
2026/09/07 21:40:57   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:40:57   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container core-bff
2026/09/07 21:40:57   21:37:17 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-858fdb6f4d from 1 to 0
2026/09/07 21:40:57   21:37:17 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulDelete - Deleted pod: rhods-dashboard-858fdb6f4d-wqtjf
2026/09/07 21:40:57   21:37:17 INFO Pod/rhods-dashboard-858fdb6f4d-wqtjf: Killing - Stopping container rhods-dashboard
2026/09/07 21:40:57   21:37:05 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe failed: Get "https://10.132.0.97:8444/healthz": dial tcp 10.132.0.97:8444: connect: connection refused
2026/09/07 21:40:57   21:36:58 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Started - Started container manager
2026/09/07 21:40:57   21:36:57 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Created - Created container: manager
2026/09/07 21:40:57   21:36:57 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3" in 1.904s (1.904s including waiting). Image size: 239132625 bytes.
2026/09/07 21:40:57   21:36:56 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Pulling - Pulling image "quay.io/rhoai/odh-model-controller-rhel9@sha256:d6712599dde6aeb175e164da80c04a2627800e83edf14ac228ac4a9fd2e2a4f3"
2026/09/07 21:40:57   21:36:55 INFO Pod/odh-model-controller-57457c4c89-xg8fl: Scheduled - Successfully assigned redhat-ods-applications/odh-model-controller-57457c4c89-xg8fl to ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   21:36:55 INFO Deployment/odh-model-controller: ScalingReplicaSet - Scaled up replica set odh-model-controller-57457c4c89 from 1 to 2
2026/09/07 21:40:57   21:36:55 INFO Pod/odh-model-controller-57457c4c89-xg8fl: AddedInterface - Add eth0 [10.134.0.112/23] from ovn-kubernetes
2026/09/07 21:40:57   21:36:55 INFO ReplicaSet/odh-model-controller-57457c4c89: SuccessfulCreate - Created pod: odh-model-controller-57457c4c89-xg8fl
2026/09/07 21:40:57   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container core-bff
2026/09/07 21:40:57   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: core-bff
2026/09/07 21:40:57   21:36:40 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 354ms (354ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:40:57   21:36:40 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:40:57   21:36:40 WARN Pod/rhods-dashboard-558d97b5db-nn7q2: Unhealthy - Readiness probe failed: Get "https://10.132.0.97:8943/healthcheck": dial tcp 10.132.0.97:8943: connect: connection refused
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container kube-rbac-proxy
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: kube-rbac-proxy
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Started - Started container rhods-dashboard
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Created - Created container: rhods-dashboard
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 416ms (416ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:40:57   21:36:39 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: AddedInterface - Add eth0 [10.132.0.102/23] from ovn-kubernetes
2026/09/07 21:40:57   21:36:38 INFO Pod/rhods-dashboard-764dcdbfcc-4chfj: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-764dcdbfcc-4chfj to ip-10-0-137-227.ec2.internal
2026/09/07 21:40:57   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:40:57   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container rhods-dashboard
2026/09/07 21:40:57   21:36:38 INFO Pod/rhods-dashboard-558d97b5db-nn7q2: Killing - Stopping container core-bff
2026/09/07 21:40:57   21:36:38 INFO ReplicaSet/rhods-dashboard-764dcdbfcc: SuccessfulCreate - Created pod: rhods-dashboard-764dcdbfcc-4chfj
2026/09/07 21:40:57   21:36:38 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-764dcdbfcc from 1 to 2
2026/09/07 21:40:57   21:36:38 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-558d97b5db from 1 to 0
2026/09/07 21:40:57   21:36:38 INFO ReplicaSet/rhods-dashboard-558d97b5db: SuccessfulDelete - Deleted pod: rhods-dashboard-558d97b5db-nn7q2
2026/09/07 21:40:57   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Pulled - Successfully pulled image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603" in 1.42s (1.42s including waiting). Image size: 172459788 bytes.
2026/09/07 21:40:57   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Started - Started container server
2026/09/07 21:40:57   21:36:37 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Created - Created container: server
2026/09/07 21:40:57   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Scheduled - Successfully assigned redhat-ods-applications/model-serving-api-7df8c9655f-xfmcz to ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: AddedInterface - Add eth0 [10.134.0.111/23] from ovn-kubernetes
2026/09/07 21:40:57   21:36:35 INFO ReplicaSet/model-serving-api-7df8c9655f: SuccessfulCreate - Created pod: model-serving-api-7df8c9655f-xfmcz
2026/09/07 21:40:57   21:36:35 INFO Pod/model-serving-api-7df8c9655f-xfmcz: Pulling - Pulling image "quay.io/rhoai/odh-model-serving-api-rhel9@sha256:32ff20440ecfc552531f609a235c4b5c9891b4d8ad89ad38dbc3dc6b0bcb7603"
2026/09/07 21:40:57   21:36:35 INFO Deployment/model-serving-api: ScalingReplicaSet - Scaled up replica set model-serving-api-7df8c9655f from 1 to 2
2026/09/07 21:40:57   21:36:30 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8943/healthcheck": dial tcp 10.132.0.101:8943: connect: connection refused
2026/09/07 21:40:57   21:36:30 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:40:57   21:36:27 WARN Pod/rhods-dashboard-858fdb6f4d-22p4c: Unhealthy - Readiness probe failed: Get "https://10.132.0.101:8444/healthz": dial tcp 10.132.0.101:8444: connect: connection refused
2026/09/07 21:40:57   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Created - Created container: manager
2026/09/07 21:40:57   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Pulled - Successfully pulled image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13" in 329ms (329ms including waiting). Image size: 235279660 bytes.
2026/09/07 21:40:57   21:36:16 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Started - Started container manager
2026/09/07 21:40:57   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Scheduled - Successfully assigned redhat-ods-applications/llmisvc-controller-manager-6df7b6c5d4-j2nvj to ip-10-0-130-155.ec2.internal
2026/09/07 21:40:57   21:36:15 INFO Deployment/llmisvc-controller-manager: ScalingReplicaSet - Scaled up replica set llmisvc-controller-manager-6df7b6c5d4 from 1 to 2
2026/09/07 21:40:57   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: AddedInterface - Add eth0 [10.134.0.110/23] from ovn-kubernetes
2026/09/07 21:40:57   21:36:15 INFO ReplicaSet/llmisvc-controller-manager-6df7b6c5d4: SuccessfulCreate - Created pod: llmisvc-controller-manager-6df7b6c5d4-j2nvj
2026/09/07 21:40:57   21:36:15 INFO Pod/llmisvc-controller-manager-6df7b6c5d4-j2nvj: Pulling - Pulling image "quay.io/rhoai/odh-kserve-llmisvc-controller-rhel9@sha256:b0b1d6155ccce66999bb4f9efbc08a8a6eff2584f1e2e2fd10342d8d36debb13"
2026/09/07 21:40:57   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
2026/09/07 21:40:57   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe failed: Get "https://10.133.0.115:8943/healthcheck": dial tcp 10.133.0.115:8943: connect: connection refused
2026/09/07 21:40:57   21:36:10 WARN Pod/rhods-dashboard-558d97b5db-5v54m: Unhealthy - Readiness probe failed: Get "https://10.133.0.115:8444/healthz": dial tcp 10.133.0.115:8444: connect: connection refused
2026/09/07 21:40:57   21:36:05 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:36:05 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:36:04 WARN ServiceMonitor/maas-controller-metrics: InvalidConfiguration - "redhat-ods-applications/maas-controller-metrics" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:36:04 WARN ServiceMonitor/odh-model-controller-metrics-monitor: InvalidConfiguration - "redhat-ods-applications/odh-model-controller-metrics-monitor" was rejected due to invalid configuration: endpoints[0]: it accesses file system via bearer token [REDACTED] which Prometheus specification prohibits
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Container image "quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:fdf969166b35abe64acbbe273caa25b24e0288849ddb36a4213c7b29e8c9d6e3" already present on machine
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container core-bff
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: core-bff
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Successfully pulled image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c" in 400ms (400ms including waiting). Image size: 181738206 bytes.
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulling - Pulling image "quay.io/rhoai/odh-core-bff-rhel9@sha256:4309e80894a3895a2156bd12fc5cb3f4043ad22d5d5cb34dd7091bb08cf9068c"
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container kube-rbac-proxy
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: kube-rbac-proxy
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Started - Started container rhods-dashboard
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Created - Created container: rhods-dashboard
2026/09/07 21:40:57   21:36:01 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulled - Successfully pulled image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e" in 324ms (324ms including waiting). Image size: 812543307 bytes.
2026/09/07 21:40:57   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Scheduled - Successfully assigned redhat-ods-applications/rhods-dashboard-764dcdbfcc-lt7q7 to ip-10-0-140-136.ec2.internal
2026/09/07 21:40:57   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: AddedInterface - Add eth0 [10.133.0.119/23] from ovn-kubernetes
2026/09/07 21:40:57   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container kube-rbac-proxy
2026/09/07 21:40:57   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container core-bff
2026/09/07 21:40:57   21:36:00 INFO Pod/rhods-dashboard-858fdb6f4d-22p4c: Killing - Stopping container rhods-dashboard
2026/09/07 21:40:57   21:36:00 INFO ReplicaSet/rhods-dashboard-764dcdbfcc: SuccessfulCreate - Created pod: rhods-dashboard-764dcdbfcc-lt7q7
2026/09/07 21:40:57   21:36:00 INFO ReplicaSet/rhods-dashboard-858fdb6f4d: SuccessfulDelete - Deleted pod: rhods-dashboard-858fdb6f4d-22p4c
2026/09/07 21:40:57   21:36:00 INFO Pod/rhods-dashboard-764dcdbfcc-lt7q7: Pulling - Pulling image "quay.io/rhoai/odh-dashboard-rhel9@sha256:b3df9a30cd1f042485bf85b287acf5374d61ce7cbbbcfa355c7fd2a777d5e62e"
2026/09/07 21:40:57   21:36:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled down replica set rhods-dashboard-858fdb6f4d from 2 to 1
2026/09/07 21:40:57   21:36:00 INFO Deployment/rhods-dashboard: ScalingReplicaSet - Scaled up replica set rhods-dashboard-764dcdbfcc from 0 to 1
2026/09/07 21:40:57   21:35:59 WARN Endpoints/odh-dashboard-autorag-ui: FailedToUpdateEndpoint - Failed to update endpoint redhat-ods-applications/odh-dashboard-autorag-ui: Operation cannot be fulfilled on endpoints "odh-dashboard-autorag-ui": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/odh-dashboard-autorag-ui, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: c737a2c7-49fc-4ca4-a807-a07ef76ed351, UID in object meta: 
2026/09/07 21:40:57   21:35:59 INFO Pod/automl-ui-74d9b97d49-stzqx: Killing - Stopping container automl-ui
2026/09/07 21:40:57   21:35:59 INFO Pod/autorag-ui-58f548cbd-8n44h: Killing - Stopping container autorag-ui
2026/09/07 21:40:57 === RESOURCE QUOTAS ===
2026/09/07 21:40:57 No resource quotas found in monitored namespaces
2026/09/07 21:40:57 Classification for group 1: infrastructure/probe-failure (code=1014, confidence=medium)
2026/09/07 21:40:57   Evidence: event Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1
FAILURE_CLASSIFICATION: {"category":"infrastructure","subcategory":"probe-failure","error_code":1014,"evidence":["event Pod/rhods-dashboard-858fdb6f4d-wqtjf: Unhealthy - Readiness probe errored and resulted in unknown state: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1"],"confidence":"medium"}
2026/09/07 21:40:57 Diagnostics complete for failed test: group 1
    controller_test.go:762: Stopping: group 1 test failed.
2026/09/07 21:40:57 Circuit breaker: failure classified as infrastructure/probe-failure (code=1014)
2026/09/07 21:40:57 Circuit breaker: infrastructure/unknown failure recorded (1/3)
    helper_test.go:156: Ensuring OCP Kueue Operator is installed (namespace=openshift-kueue-operator, name=kueue-operator).
    helper_test.go:156: Creating test namespace (test-kueue-no-queues-dafiv9pve3cs7m7v0qgg) with Kueue management annotation.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) to verify default Kueue resource is created correctly.
    helper_test.go:156: Creating Kueue ConfigMap (kueue-manager-config) in namespace (redhat-ods-applications).
    helper_test.go:156: Updating DSC to set Kueue to Unmanaged with autoCreateQueues=false (default).
    helper_test.go:156: Verifying Kueue configuration (cluster) is still created.
    helper_test.go:156: Verifying no ClusterQueue, ResourceFlavor or LocalQueue is created (autoCreateQueues=false).
    helper_test.go:156: Cleaning up Kueue test resources.
    helper_test.go:156: Cleaning up Kueue cluster-scoped resources
    helper_test.go:156: Will attempt to delete 3 Kueue resources
    helper_test.go:156: Attempting to delete ClusterQueue /default
    helper_test.go:156: Successfully processed deletion of ClusterQueue /default
    helper_test.go:156: Attempting to delete ResourceFlavor /default-flavor
    helper_test.go:156: Successfully processed deletion of ResourceFlavor /default-flavor
    helper_test.go:156: Attempting to delete Kueue /cluster
    helper_test.go:156: Successfully processed deletion of Kueue /cluster
    test_context_test.go:1111: cleanup: best-effort delete failed for /, Kind= redhat-ods-applications/kueue-manager-config: Object 'Kind' is missing in 'unstructured object has no kind'
    resource_applier_test.go:211: 
        Timed out after 600.001s.
        The function passed to Eventually failed at /workspace/tests/e2e/resource_applier_test.go:287 with:
        Error occurred while applying the resource 'default-dsc' of kind 'DataScienceCluster': Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
        Unexpected error:
            <*url.Error | 0x34e0256e8db0>: 
            Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
            {
                Op: "Get",
                URL: "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc",
                Err: <*net.OpError | 0x34e0262588c0>{
                    Op: "dial",
                    Net: "tcp",
                    Source: nil,
                    Addr: nil,
                    Err: <*net.DNSError | 0x34e026258870>{
                        UnwrapErr: nil,
                        Err: "no such host",
                        Name: "a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com",
                        Server: "172.30.0.10:53",
                        IsTimeout: false,
                        IsTemporary: false,
                        IsNotFound: true,
                    },
                },
            }
        occurred
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/07 21:52:31 TEST FAILURE DETECTED: group 3
2026/09/07 21:52:31 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 3) ===
2026/09/07 21:52:31 Failed to get DSCI: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 Failed to discover operator deployment in namespace redhat-ods-operator: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === CLUSTER STATE ===
2026/09/07 21:52:31 Failed to collect node data: failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === DEPLOYMENTS ===
2026/09/07 21:52:31 Failed to collect deployment data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-applications/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/kube-system/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === PODS ===
2026/09/07 21:52:31 Failed to collect pod data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/pods": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/pods": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === OPERATOR STATUS ===
2026/09/07 21:52:31 Failed to collect operator data: operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === DSCI STATUS ===
2026/09/07 21:52:31 Failed to collect DSCI data: failed to get CR: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === DSC STATUS ===
2026/09/07 21:52:31 Failed to collect DSC data: failed to get CR: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === RECENT EVENTS ===
2026/09/07 21:52:31 Failed to collect event data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/events?limit=500": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/events?limit=500": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 === RESOURCE QUOTAS ===
2026/09/07 21:52:31 Failed to collect quota data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/resourcequotas": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/resourcequotas": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 21:52:31 Classification for group 3: unknown/unclassifiable (code=3000, confidence=low)
2026/09/07 21:52:31   Evidence: no matching classification rule
FAILURE_CLASSIFICATION: {"category":"unknown","subcategory":"unclassifiable","error_code":3000,"evidence":["no matching classification rule"],"confidence":"low"}
2026/09/07 21:52:31 Diagnostics complete for failed test: group 3
    controller_test.go:762: Stopping: group 3 test failed.
2026/09/07 21:52:31 Circuit breaker: failure classified as unknown/unclassifiable (code=3000)
2026/09/07 21:52:31 Circuit breaker: infrastructure/unknown failure recorded (2/3)
    test_context_test.go:120: Warning: cluster.Init failed: failed to get server groups: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
    resource_applier_test.go:211: 
        Timed out after 600.008s.
        The function passed to Eventually failed at /workspace/tests/e2e/resource_applier_test.go:287 with:
        Error occurred while applying the resource 'default-dsc' of kind 'DataScienceCluster': failed to get server groups: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
        Unexpected error:
            <*fmt.wrapError | 0x34e027253920>: 
            failed to get server groups: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
            {
                msg: "failed to get server groups: Get \"https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api\": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host",
                err: <*url.Error | 0x34e026592b40>{
                    Op: "Get",
                    URL: "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api",
                    Err: <*net.OpError | 0x34e025eb3720>{
                        Op: "dial",
                        Net: "tcp",
                        Source: nil,
                        Addr: nil,
                        Err: <*net.DNSError | 0x34e025eb36d0>{
                            UnwrapErr: nil,
                            Err: "no such host",
                            Name: "a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com",
                            Server: "172.30.0.10:53",
                            IsTimeout: false,
                            IsTemporary: false,
                            IsNotFound: true,
                        },
                    },
                },
            }
        occurred
    testing.go:1913: test executed panic(nil) or runtime.Goexit: subtest may have called FailNow on a parent test
2026/09/07 22:02:31 TEST FAILURE DETECTED: group 4
2026/09/07 22:02:31 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:group 4) ===
2026/09/07 22:02:31 Failed to get DSCI: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 Failed to discover operator deployment in namespace redhat-ods-operator: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === CLUSTER STATE ===
2026/09/07 22:02:31 Failed to collect node data: failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === DEPLOYMENTS ===
2026/09/07 22:02:31 Failed to collect deployment data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-applications/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/kube-system/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === PODS ===
2026/09/07 22:02:31 Failed to collect pod data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/pods": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/pods": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === OPERATOR STATUS ===
2026/09/07 22:02:31 Failed to collect operator data: operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === DSCI STATUS ===
2026/09/07 22:02:31 Failed to collect DSCI data: failed to get CR: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === DSC STATUS ===
2026/09/07 22:02:31 Failed to collect DSC data: failed to get CR: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === RECENT EVENTS ===
2026/09/07 22:02:31 Failed to collect event data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/events?limit=500": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/events?limit=500": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === RESOURCE QUOTAS ===
2026/09/07 22:02:31 Failed to collect quota data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/resourcequotas": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/resourcequotas": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 Classification for group 4: unknown/unclassifiable (code=3000, confidence=low)
2026/09/07 22:02:31   Evidence: no matching classification rule
FAILURE_CLASSIFICATION: {"category":"unknown","subcategory":"unclassifiable","error_code":3000,"evidence":["no matching classification rule"],"confidence":"low"}
2026/09/07 22:02:31 Diagnostics complete for failed test: group 4
    controller_test.go:762: Stopping: group 4 test failed.
2026/09/07 22:02:31 Circuit breaker: failure classified as unknown/unclassifiable (code=3000)
2026/09/07 22:02:31 Circuit breaker: infrastructure/unknown failure recorded (3/3)
2026/09/07 22:02:31 Circuit breaker: threshold reached (3 failures), running confirming health check...
2026/09/07 22:02:31 Failed to get DSCI: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 Failed to discover operator deployment in namespace redhat-ods-operator: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 CIRCUIT BREAKER TRIPPED: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]
2026/09/07 22:02:31 TEST FAILURE DETECTED: components
2026/09/07 22:02:31 === RUNNING CLUSTER DIAGNOSTICS (triggered by test:components) ===
2026/09/07 22:02:31 Failed to get DSCI: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 Failed to discover operator deployment in namespace redhat-ods-operator: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === CLUSTER STATE ===
2026/09/07 22:02:31 Failed to collect node data: failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === DEPLOYMENTS ===
2026/09/07 22:02:31 Failed to collect deployment data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-applications/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/kube-system/deployments": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === PODS ===
2026/09/07 22:02:31 Failed to collect pod data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/pods": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/pods": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === OPERATOR STATUS ===
2026/09/07 22:02:31 Failed to collect operator data: operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === DSCI STATUS ===
2026/09/07 22:02:31 Failed to collect DSCI data: failed to get CR: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/dscinitialization.opendatahub.io/v2/dscinitializations/default-dsci": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === DSC STATUS ===
2026/09/07 22:02:31 Failed to collect DSC data: failed to get CR: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/datasciencecluster.opendatahub.io/v2/datascienceclusters/default-dsc": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === RECENT EVENTS ===
2026/09/07 22:02:31 Failed to collect event data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/events?limit=500": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/events?limit=500": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 === RESOURCE QUOTAS ===
2026/09/07 22:02:31 Failed to collect quota data: redhat-ods-applications: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/redhat-ods-applications/resourcequotas": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; kube-system: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/kube-system/resourcequotas": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host
2026/09/07 22:02:31 Classification for components: unknown/unclassifiable (code=3000, confidence=low)
2026/09/07 22:02:31   Evidence: no matching classification rule
FAILURE_CLASSIFICATION: {"category":"unknown","subcategory":"unclassifiable","error_code":3000,"evidence":["no matching classification rule"],"confidence":"low"}
2026/09/07 22:02:31 Diagnostics complete for failed test: components
    controller_test.go:490: Stopping: components test failed.
Deletion Policy: Never. Skipping deletion tests.
2026/09/07 22:02:31 === CIRCUIT BREAKER SUMMARY ===
2026/09/07 22:02:31 Tripped 1 time(s)
2026/09/07 22:02:31 Reason: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]
2026/09/07 22:02:31 ===============================
    controller_test.go:428: Circuit breaker tripped: Infrastructure failure detected after 3 infrastructure/unknown failures. Cluster health issues: [failed to list nodes: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/api/v1/nodes": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host; operator deployment not found: Get "https://a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com:6443/apis/apps/v1/namespaces/redhat-ods-operator/deployments/opendatahub-operator-controller-manager": dial tcp: lookup a84c11ecdafd64e5883a789ff49c02ad-0f6551d610185b95.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host]
    testgroup_test.go:119: 
        	Error Trace:	/workspace/tests/e2e/testgroup_test.go:119
        	Error:      	[]string{} does not contain "auth"
        	Test:       	TestRunExcluding/runs_all_except_excluded]]></system-out>
	</testsuite>
</testsuites>
