('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')test_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m'], prompt='KServe is a', service_name=... {'name': 'model-fb-opt-125m-auth-enabled-89f54b63'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.llminferenceservice @pytest.mark.auth @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="auth-enabled-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, ], id="auth-enabled-default", ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_auth_enabled_requires_token(test_case: TestCase): # noqa: F811 """ Test that when auth is enabled (default): - Requests WITH valid token succeed - Requests WITHOUT token are rejected (401/403) """ inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) service_name = test_case.llm_service.metadata.name sa_name = f"{service_name}-test-sa" test_failed = False # Enable auth for this test if not test_case.llm_service.metadata.annotations: test_case.llm_service.metadata.annotations = {} test_case.llm_service.metadata.annotations[ "security.opendatahub.io/enable-auth" ] = "true" try: # Create LLMInferenceService create_llmisvc(kserve_client, test_case.llm_service) wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) # Create ServiceAccount with get+post access (required for inference-access and endpoint-access) token = create_service_account_with_inference_access( kserve_client, sa_name, service_name ) service_url = get_llm_service_url(kserve_client, test_case.llm_service) completion_url = f"{service_url}/v1/completions" test_payload = { "model": test_case.model_name, "prompt": test_case.prompt, "max_tokens": test_case.max_tokens, } # Test 1: Request WITHOUT token should fail logger.info("Testing request WITHOUT token (should fail)") response_no_token = requests.post( completion_url, headers={"Content-Type": "application/json"}, json=test_payload, timeout=30, ) assert response_no_token.status_code in [ 401, 403, ], ( f"Expected 401/403 without token, got {response_no_token.status_code}: {response_no_token.text}" ) logger.info( f"✅ Request without token rejected: {response_no_token.status_code}" ) # Test 2: Request WITH valid token should succeed. # Retry to handle propagation delay — RBAC changes and Authorino auth policy # caches may not reflect new bindings immediately after creation. logger.info("Testing request WITH valid token (should succeed)") token_headers = { "Content-Type": "application/json", "Authorization": f"Bearer {token}", } response_with_token = None for attempt in range(24): # up to ~120s response_with_token = requests.post( completion_url, headers=token_headers, json=test_payload, timeout=test_case.response_timeout, ) if response_with_token.status_code == 200: break if response_with_token.status_code in [401, 403]: logger.info( f"Attempt {attempt + 1}: got {response_with_token.status_code}, " "waiting for RBAC propagation..." ) time.sleep(5) else: break > assert response_with_token.status_code == 200, ( f"Expected 200 with token, got {response_with_token.status_code}: {response_with_token.text}" ) E AssertionError: Expected 200 with token, got 403: E assert 403 == 200 E + where 403 = <Response [403]>.status_code llmisvc/test_llm_auth.py:336: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-hpa-deploy", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_hpa_deployment(test_case: TestCase): """HPA + Deployment: VA and HPA exist; pods scale up under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:540: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d21069850> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:480: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d21069850>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-hpa-4c186bcf'}, {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-20T10:24:15.869788', start_time = 1784543055.87021 duration = 900.8121633529663, timestamp_end = '2026-07-20T10:39:16.682376' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d21069850> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': None, ...toscale-hpa-4c186bcf'}, {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1255: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d21052700> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1266: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1250: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_keda @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-keda", ], prompt="KServe is a", service_name="autoscale-keda-deploy", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_keda_deployment(test_case: TestCase): """KEDA + Deployment: VA and ScaledObject exist; no HPA; pods scale up under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:604: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d20f71210> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:480: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d20f71210>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-ked-101f2a9d'}, {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-20T10:39:22.870344', start_time = 1784543962.8706155 duration = 900.0523600578308, timestamp_end = '2026-07-20T10:54:22.922983' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d20f71210> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': None, ...toscale-ked-101f2a9d'}, {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1255: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d20616a20> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1266: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1250: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-lws", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-hpa-lws", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_hpa_lws(test_case: TestCase): """HPA + LWS: VA and HPA exist; pods scale under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:668: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d205dff50> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:480: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d205dff50>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...ale-hpa-b29acdba'}, {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-20T10:54:28.467008', start_time = 1784544868.4673026 duration = 900.705540895462, timestamp_end = '2026-07-20T11:09:29.172846' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d205dff50> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': None, ...autoscale-hpa-b29acdba'}, {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1255: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d206177e0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1266: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:54:44Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1250: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_keda @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-lws", "prometheus-scrape", "scaling-keda", ], prompt="KServe is a", service_name="autoscale-keda-lws", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_keda_lws(test_case: TestCase): """KEDA + LWS: VA and ScaledObject exist; pods scale under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:726: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d20523890> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:480: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d20523890>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-ked-231d315d'}, {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-20T11:09:35.035005', start_time = 1784545775.0353525 duration = 900.581330537796, timestamp_end = '2026-07-20T11:24:35.616685' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d20523890> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': None, ...toscale-ked-231d315d'}, {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1255: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d20616ca0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1266: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() conditions = status["conditions"] for condition in conditions: if condition.get("status") == "True": got_true_conditions.add(condition.get("type")) missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T11:09:53Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind "VariantAutoscaling" in version "llmd.ai/v1alpha1"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1250: AssertionError