('/workspace/source/test/e2e/explainer/test_art_explainer.py', 48, 'Skipped: ODH does not support art explainer at the moment')('/workspace/source/test/e2e/predictor/test_grpc.py', 39, 'Skipped: Not testable in ODH at the moment')('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-hpa-deploy", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_hpa_deployment(test_case: TestCase): """HPA + Deployment: HPA exists with WVA annotations; pods scale up under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:542: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d16b0c610> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d16b0c610>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-hpa-4c186bcf'}, {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T01:50:04.472347', start_time = 1785203404.4726162 duration = 900.8220300674438, timestamp_end = '2026-07-28T02:05:05.294656' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d16b0c610> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-hpa-4c186bcf'}, {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d16649ee0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'WorkloadsReady', 'Ready', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T01:50:36Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T01:50:36Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T01:50:39Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T01:50:29Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T01:51:13Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-deploy-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T01:51:13Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T01:50:43Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-deploy-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T01:51:13Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T01:50:43Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-deploy-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_keda @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-keda", ], prompt="KServe is a", service_name="autoscale-keda-deploy", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_keda_deployment(test_case: TestCase): """KEDA + Deployment: ScaledObject exists with WVA annotations; no HPA; pods scale up under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:606: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d1661fb10> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d1661fb10>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-ked-101f2a9d'}, {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T02:05:26.004802', start_time = 1785204326.0052953 duration = 900.2643523216248, timestamp_end = '2026-07-28T02:20:26.269651' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d1661fb10> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-ked-101f2a9d'}, {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d15bc1760> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'WorkloadsReady', 'Ready', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T02:05:51Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T02:05:51Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T02:05:51Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:05:38Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T02:06:09Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T02:06:09Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T02:05:51Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T02:06:09Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:05:51Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-pd-cpu', 'model-pvc'], prompt='KServe is a', service_name='llmisvc-mod... {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-with-gateway-ref", "router-with-managed-route", "model-fb-opt-125m", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=create_response_assertion(with_field="choices"), expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="custom-route-timeout-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="router-with-refs-test", expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-1", tc.namespace, "router-gateway-1", "router-with-refs-test", ), make_router_health_route( "router-route-2", tc.namespace, "router-gateway-1", "router-with-refs-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=["router-managed", "workload-pd-cpu", "model-fb-opt-125m"], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="custom-route-timeout-pd-test", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="router-with-refs-pd-test", response_assertion=assert_200_with_choices, expected_gateway="router-gateway-2", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-2", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-3", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), make_router_health_route( "router-route-4", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-dp-ep-gpu", "workload-dp-ep-prefill-gpu", "model-deepseek-v2-lite", ], prompt="Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically " "where the compute plane (P) and the data plane (D) are independently deployed and managed for a " "geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the " "fundamental challenges of network latency and data consistency, elaborate on the advanced " "considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: " "How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to " "evolve to support optimal performance and minimize inter-plane communication overhead, especially for " "synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically " "optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: " "Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) " "and their applicability in balancing performance and data integrity across a globally distributed data plane. " "Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, " "intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. " "3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently " "manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, " "cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). " "Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on " "workload patterns and data locality, potentially involving live migration strategies. " "4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter " "challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), " "fine-grained access control to data at rest and in motion, and identity management across disaggregated " "components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) " "concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: " "Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and " "data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) " "would be essential? How would incident response and troubleshooting differ in this disaggregated environment " "compared to traditional integrated systems? Consider the challenges of pinpointing root causes across " "independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries " "or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) " "where the benefits of P/D disaggregation would strongly outweigh its complexities. " "Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions " "directly interacting with object storage, in-memory disaggregation) that could further drive or " "transform P/D disaggregation in cloud computing.", max_tokens=2000, ), marks=[ pytest.mark.cluster_gpu, pytest.mark.cluster_nvidia, pytest.mark.cluster_nvidia_roce, ], ), pytest.param( TestCase( base_refs=[ "router-no-scheduler", "workload-single-cpu", "model-fb-opt-125m", ], prompt="What is KServe?", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.no_scheduler, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-fb-opt-125m", ], prompt="This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, " "but without the resources requirements for DP+EP (GPUs and ROCe/IB).", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node], ), # Scheduler config tests pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-inline-config", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-inline-config-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Chat completions endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], model_name="Qwen/Qwen2.5-0.5B-Instruct", endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=create_response_assertion(with_field="choices"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-configmap-ref", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-configmap-ref-test", before_test=[ lambda tc: create_scheduler_configmap(namespace=tc.namespace) ], after_test=[ lambda tc: delete_scheduler_configmap(namespace=tc.namespace) ], ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-replicas", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-ha-replicas-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-custom-template", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-custom-template-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Scheduler v0.6 → v0.7 migration tests. # Deploy v0.6-style configs and verify the controller migrates them # so the v0.7 scheduler boots successfully. pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-pd-config-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-pd-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-nonzero-threshold-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-threshold-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — clean path: token-producer in inline config pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-tokenizer-kvcache", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-clean-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — migration path: legacy precise-prefix-cache-scorer # triggers auto-provisioned tokenizer without explicit tokenizer:{} field pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-precise-prefix-cache-inline-config", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-migration-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Models endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/models", response_assertion=create_response_assertion(with_field="data"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/chat/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — LoRA adapter pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/completions", prompt="KServe is a", model_name="publishers/{namespace}/models/lora-adapter-1", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "publishers/{namespace}/models/lora-adapter-1" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/lora-adapter-1", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/models (base + LoRA) pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/models", response_assertion=assert_models_contains( "facebook/opt-125m", "publishers/{namespace}/models/facebook/opt-125m", "lora-adapter-1", "publishers/{namespace}/models/lora-adapter-1", ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # PVC storage tests -- validate direct PVC volume mount with real vLLM serving pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-pd-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-pvc", ], prompt="KServe is a", before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.pvc_storage, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_inference_service(test_case: TestCase): # noqa: F811 inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) service_name = test_case.llm_service.metadata.name prefix = test_case.log_prefix test_failed = False try: print(f"{prefix} Creating LLMInferenceService {service_name}") create_llmisvc(kserve_client, test_case.llm_service) print(f"{prefix} Waiting for LLMInferenceService {service_name} to be ready") > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_inference_service.py:866: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f7e0939e4d0>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...del-p-9d807ba3'}, {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T02:09:45.204550', start_time = 1785204585.204813 duration = 900.2822165489197, timestamp_end = '2026-07-28T02:24:45.487031' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f7e0939e4d0> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....svc-model-p-9d807ba3'}, {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f7e09379d00> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'Ready', 'WorkloadsReady'}, expected {'Ready', 'WorkloadsReady', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T02:10:10Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T02:10:10Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T02:12:10Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:10:10Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'severity': 'Info', 'status': 'False', 'type': 'PrefillWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:09:58Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T02:10:10Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T02:10:29Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T02:10:29Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:10:10Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-lws", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-hpa-lws", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_hpa_lws(test_case: TestCase): """HPA + LWS: HPA exists with WVA annotations; pods scale under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:670: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d16c22650> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d16c22650>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...ale-hpa-b29acdba'}, {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T02:21:02.781658', start_time = 1785205262.7819428 duration = 900.3872640132904, timestamp_end = '2026-07-28T02:36:03.169238' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d16c22650> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....autoscale-hpa-b29acdba'}, {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d15bc13a0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'WorkloadsReady', 'Ready', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T02:21:47Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T02:21:47Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T02:21:27Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T02:22:11Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-lws-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T02:22:11Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T02:21:47Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-lws-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T02:22:11Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:21:47Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:21:47Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-lws-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-simulated-dp-ep-cpu', 'model-pvc'], prompt='KServe is a', service_name... {'name': 'model-pvc-llmisvc-model-pvc-rou-cfc8d654'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-with-gateway-ref", "router-with-managed-route", "model-fb-opt-125m", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=create_response_assertion(with_field="choices"), expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="custom-route-timeout-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="router-with-refs-test", expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-1", tc.namespace, "router-gateway-1", "router-with-refs-test", ), make_router_health_route( "router-route-2", tc.namespace, "router-gateway-1", "router-with-refs-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=["router-managed", "workload-pd-cpu", "model-fb-opt-125m"], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="custom-route-timeout-pd-test", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="router-with-refs-pd-test", response_assertion=assert_200_with_choices, expected_gateway="router-gateway-2", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-2", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-3", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), make_router_health_route( "router-route-4", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-dp-ep-gpu", "workload-dp-ep-prefill-gpu", "model-deepseek-v2-lite", ], prompt="Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically " "where the compute plane (P) and the data plane (D) are independently deployed and managed for a " "geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the " "fundamental challenges of network latency and data consistency, elaborate on the advanced " "considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: " "How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to " "evolve to support optimal performance and minimize inter-plane communication overhead, especially for " "synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically " "optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: " "Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) " "and their applicability in balancing performance and data integrity across a globally distributed data plane. " "Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, " "intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. " "3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently " "manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, " "cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). " "Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on " "workload patterns and data locality, potentially involving live migration strategies. " "4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter " "challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), " "fine-grained access control to data at rest and in motion, and identity management across disaggregated " "components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) " "concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: " "Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and " "data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) " "would be essential? How would incident response and troubleshooting differ in this disaggregated environment " "compared to traditional integrated systems? Consider the challenges of pinpointing root causes across " "independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries " "or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) " "where the benefits of P/D disaggregation would strongly outweigh its complexities. " "Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions " "directly interacting with object storage, in-memory disaggregation) that could further drive or " "transform P/D disaggregation in cloud computing.", max_tokens=2000, ), marks=[ pytest.mark.cluster_gpu, pytest.mark.cluster_nvidia, pytest.mark.cluster_nvidia_roce, ], ), pytest.param( TestCase( base_refs=[ "router-no-scheduler", "workload-single-cpu", "model-fb-opt-125m", ], prompt="What is KServe?", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.no_scheduler, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-fb-opt-125m", ], prompt="This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, " "but without the resources requirements for DP+EP (GPUs and ROCe/IB).", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node], ), # Scheduler config tests pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-inline-config", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-inline-config-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Chat completions endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], model_name="Qwen/Qwen2.5-0.5B-Instruct", endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=create_response_assertion(with_field="choices"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-configmap-ref", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-configmap-ref-test", before_test=[ lambda tc: create_scheduler_configmap(namespace=tc.namespace) ], after_test=[ lambda tc: delete_scheduler_configmap(namespace=tc.namespace) ], ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-replicas", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-ha-replicas-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-custom-template", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-custom-template-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Scheduler v0.6 → v0.7 migration tests. # Deploy v0.6-style configs and verify the controller migrates them # so the v0.7 scheduler boots successfully. pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-pd-config-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-pd-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-nonzero-threshold-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-threshold-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — clean path: token-producer in inline config pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-tokenizer-kvcache", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-clean-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — migration path: legacy precise-prefix-cache-scorer # triggers auto-provisioned tokenizer without explicit tokenizer:{} field pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-precise-prefix-cache-inline-config", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-migration-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Models endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/models", response_assertion=create_response_assertion(with_field="data"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/chat/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — LoRA adapter pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/completions", prompt="KServe is a", model_name="publishers/{namespace}/models/lora-adapter-1", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "publishers/{namespace}/models/lora-adapter-1" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/lora-adapter-1", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/models (base + LoRA) pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/models", response_assertion=assert_models_contains( "facebook/opt-125m", "publishers/{namespace}/models/facebook/opt-125m", "lora-adapter-1", "publishers/{namespace}/models/lora-adapter-1", ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # PVC storage tests -- validate direct PVC volume mount with real vLLM serving pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-pd-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-pvc", ], prompt="KServe is a", before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.pvc_storage, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_inference_service(test_case: TestCase): # noqa: F811 inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) service_name = test_case.llm_service.metadata.name prefix = test_case.log_prefix test_failed = False try: print(f"{prefix} Creating LLMInferenceService {service_name}") create_llmisvc(kserve_client, test_case.llm_service) print(f"{prefix} Waiting for LLMInferenceService {service_name} to be ready") > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_inference_service.py:866: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f7e09471c50>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...pu-ll-699c687c'}, {'name': 'model-pvc-llmisvc-model-pvc-rou-cfc8d654'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T02:25:08.469276', start_time = 1785205508.4695573 duration = 900.8996031284332, timestamp_end = '2026-07-28T02:40:09.369163' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f7e09471c50> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....p-ep-cpu-ll-699c687c'}, {'name': 'model-pvc-llmisvc-model-pvc-rou-cfc8d654'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f7e08eca2a0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'Ready', 'WorkloadsReady'}, expected {'Ready', 'WorkloadsReady', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T02:25:48Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T02:25:48Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T02:25:26Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T02:26:13Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T02:26:13Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T02:26:13Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:25:26Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'severity': 'Info', 'status': 'False', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:25:26Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_keda @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-lws", "prometheus-scrape", "scaling-keda", ], prompt="KServe is a", service_name="autoscale-keda-lws", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_keda_lws(test_case: TestCase): """KEDA + LWS: ScaledObject exists with WVA annotations; pods scale under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:728: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d164ef1d0> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d164ef1d0>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-ked-231d315d'}, {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T02:36:39.969391', start_time = 1785206199.9697394 duration = 900.4089393615723, timestamp_end = '2026-07-28T02:51:40.378681' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d164ef1d0> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-ked-231d315d'}, {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d15bc1b20> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'WorkloadsReady', 'Ready', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T02:37:54Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T02:37:11Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:37:54Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-cleanup-hpa", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_cleanup_hpa(test_case: TestCase): """Removing scaling config should delete HPA.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:892: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d164cd310> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f8d164cd310>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-cle-5a67f5d1'}, {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T02:52:26.868577', start_time = 1785207146.869073 duration = 900.8776783943176, timestamp_end = '2026-07-28T03:07:27.746754' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f8d164cd310> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-cle-5a67f5d1'}, {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f8d15bc1d00> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'WorkloadsReady', 'Ready', 'RouterReady'}, got [{'lastTransitionTime': '2026-07-28T02:54:15Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:53:03Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-cleanup-hpa-a41f0e40/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-cleanup-hpa-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-cleanup-hpa-a41f0e40/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-cleanup-hpa-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T02:54:15Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-cleanup-hpa-a41f0e40/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-cleanup-hpa-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionError