('/workspace/source/test/e2e/explainer/test_art_explainer.py', 48, 'Skipped: ODH does not support art explainer at the moment')('/workspace/source/test/e2e/predictor/test_grpc.py', 39, 'Skipped: Not testable in ODH at the moment')('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-hpa-deploy", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_hpa_deployment(test_case: TestCase): """HPA + Deployment: HPA exists with WVA annotations; pods scale up under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:542: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd08a58d0> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f0dd08a58d0>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-hpa-4c186bcf'}, {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T15:58:21.578162', start_time = 1785254301.5784934 duration = 900.7017710208893, timestamp_end = '2026-07-28T16:13:22.280286' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd08a58d0> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-hpa-4c186bcf'}, {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f0dd0031ee0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'RouterReady', 'WorkloadsReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-28T15:59:06Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T15:59:06Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T15:59:06Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T15:58:53Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T15:59:24Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-deploy-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T15:59:24Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T15:59:07Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-deploy-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T15:59:24Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T15:59:07Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-deploy-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_keda @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-keda", ], prompt="KServe is a", service_name="autoscale-keda-deploy", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_keda_deployment(test_case: TestCase): """KEDA + Deployment: ScaledObject exists with WVA annotations; no HPA; pods scale up under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:606: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd06593d0> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f0dd06593d0>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-ked-101f2a9d'}, {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T16:13:48.489941', start_time = 1785255228.4902089 duration = 900.0904982089996, timestamp_end = '2026-07-28T16:28:48.580711' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd06593d0> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-ked-101f2a9d'}, {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f0dd01f2ac0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'RouterReady', 'WorkloadsReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-28T16:14:31Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T16:14:31Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T16:14:31Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:14:16Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T16:14:47Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T16:14:47Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T16:14:31Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T16:14:47Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:14:31Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-pd-cpu', 'model-pvc'], prompt='KServe is a', service_name='llmisvc-mod... {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-with-gateway-ref", "router-with-managed-route", "model-fb-opt-125m", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=create_response_assertion(with_field="choices"), expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="custom-route-timeout-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="router-with-refs-test", expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-1", tc.namespace, "router-gateway-1", "router-with-refs-test", ), make_router_health_route( "router-route-2", tc.namespace, "router-gateway-1", "router-with-refs-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=["router-managed", "workload-pd-cpu", "model-fb-opt-125m"], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="custom-route-timeout-pd-test", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="router-with-refs-pd-test", response_assertion=assert_200_with_choices, expected_gateway="router-gateway-2", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-2", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-3", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), make_router_health_route( "router-route-4", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-dp-ep-gpu", "workload-dp-ep-prefill-gpu", "model-deepseek-v2-lite", ], prompt="Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically " "where the compute plane (P) and the data plane (D) are independently deployed and managed for a " "geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the " "fundamental challenges of network latency and data consistency, elaborate on the advanced " "considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: " "How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to " "evolve to support optimal performance and minimize inter-plane communication overhead, especially for " "synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically " "optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: " "Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) " "and their applicability in balancing performance and data integrity across a globally distributed data plane. " "Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, " "intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. " "3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently " "manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, " "cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). " "Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on " "workload patterns and data locality, potentially involving live migration strategies. " "4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter " "challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), " "fine-grained access control to data at rest and in motion, and identity management across disaggregated " "components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) " "concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: " "Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and " "data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) " "would be essential? How would incident response and troubleshooting differ in this disaggregated environment " "compared to traditional integrated systems? Consider the challenges of pinpointing root causes across " "independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries " "or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) " "where the benefits of P/D disaggregation would strongly outweigh its complexities. " "Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions " "directly interacting with object storage, in-memory disaggregation) that could further drive or " "transform P/D disaggregation in cloud computing.", max_tokens=2000, ), marks=[ pytest.mark.cluster_gpu, pytest.mark.cluster_nvidia, pytest.mark.cluster_nvidia_roce, ], ), pytest.param( TestCase( base_refs=[ "router-no-scheduler", "workload-single-cpu", "model-fb-opt-125m", ], prompt="What is KServe?", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.no_scheduler, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-fb-opt-125m", ], prompt="This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, " "but without the resources requirements for DP+EP (GPUs and ROCe/IB).", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node], ), # Scheduler config tests pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-inline-config", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-inline-config-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Chat completions endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], model_name="Qwen/Qwen2.5-0.5B-Instruct", endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=create_response_assertion(with_field="choices"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-configmap-ref", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-configmap-ref-test", before_test=[ lambda tc: create_scheduler_configmap(namespace=tc.namespace) ], after_test=[ lambda tc: delete_scheduler_configmap(namespace=tc.namespace) ], ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-replicas", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-ha-replicas-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-custom-template", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-custom-template-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Scheduler v0.6 → v0.7 migration tests. # Deploy v0.6-style configs and verify the controller migrates them # so the v0.7 scheduler boots successfully. pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-pd-config-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-pd-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-nonzero-threshold-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-threshold-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — clean path: token-producer in inline config pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-tokenizer-kvcache", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-clean-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — migration path: legacy precise-prefix-cache-scorer # triggers auto-provisioned tokenizer without explicit tokenizer:{} field pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-precise-prefix-cache-inline-config", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-migration-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Models endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/models", response_assertion=create_response_assertion(with_field="data"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/chat/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — LoRA adapter pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/completions", prompt="KServe is a", model_name="publishers/{namespace}/models/lora-adapter-1", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "publishers/{namespace}/models/lora-adapter-1" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/lora-adapter-1", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/models (base + LoRA) pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/models", response_assertion=assert_models_contains( "facebook/opt-125m", "publishers/{namespace}/models/facebook/opt-125m", "lora-adapter-1", "publishers/{namespace}/models/lora-adapter-1", ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # PVC storage tests -- validate direct PVC volume mount with real vLLM serving pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-pd-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-pvc", ], prompt="KServe is a", before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.pvc_storage, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_inference_service(test_case: TestCase): # noqa: F811 inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) service_name = test_case.llm_service.metadata.name prefix = test_case.log_prefix test_failed = False try: print(f"{prefix} Creating LLMInferenceService {service_name}") create_llmisvc(kserve_client, test_case.llm_service) print(f"{prefix} Waiting for LLMInferenceService {service_name} to be ready") > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_inference_service.py:866: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7fc202111d50>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...del-p-9d807ba3'}, {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T16:18:26.479986', start_time = 1785255506.4802804 duration = 900.4899139404297, timestamp_end = '2026-07-28T16:33:26.970197' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc202111d50> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....svc-model-p-9d807ba3'}, {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7fc2021e1440> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'RouterReady', 'WorkloadsReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-28T16:18:55Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T16:18:55Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T16:20:50Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:18:55Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'severity': 'Info', 'status': 'False', 'type': 'PrefillWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:18:44Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T16:18:55Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T16:19:35Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T16:19:35Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:18:55Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-lws", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-hpa-lws", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_hpa_lws(test_case: TestCase): """HPA + LWS: HPA exists with WVA annotations; pods scale under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:670: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd1488390> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f0dd1488390>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...ale-hpa-b29acdba'}, {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T16:29:10.602219', start_time = 1785256150.6025183 duration = 900.720700263977, timestamp_end = '2026-07-28T16:44:11.323225' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd1488390> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....autoscale-hpa-b29acdba'}, {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f0dcb7e2200> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'RouterReady', 'WorkloadsReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-28T16:29:33Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T16:29:33Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T16:29:18Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T16:29:53Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-lws-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T16:29:53Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T16:29:33Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-lws-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T16:29:53Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:29:33Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:29:33Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-hpa-lws-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_keda @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-lws", "prometheus-scrape", "scaling-keda", ], prompt="KServe is a", service_name="autoscale-keda-lws", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_keda_lws(test_case: TestCase): """KEDA + LWS: ScaledObject exists with WVA annotations; pods scale under load.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:728: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd07ca190> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f0dd07ca190>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-ked-231d315d'}, {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T16:44:38.171452', start_time = 1785257078.1717303 duration = 900.9549708366394, timestamp_end = '2026-07-28T16:59:39.126704' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd07ca190> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-ked-231d315d'}, {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f0dcb7e2340> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'RouterReady', 'WorkloadsReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-28T16:45:48Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T16:45:02Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T16:45:48Z', 'message': 'failed to ensure HPA is correctly created for ScaledObject: error parsing prometheus metadata: error parsing prometheus metadata: bearer token=<empty> is required when bearer auth is enabled', 'reason': 'ScaledObjectCheckFailed', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrortest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.autoscaling_hpa @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator-no-replicas", "prometheus-scrape", "scaling-hpa", ], prompt="KServe is a", service_name="autoscale-cleanup-hpa", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_autoscaling_cleanup_hpa(test_case: TestCase): """Removing scaling config should delete HPA.""" inject_k8s_proxy() kserve_client = _new_kserve_client() service_name = test_case.llm_service.metadata.name ns = test_case.namespace try: > _create_and_wait(kserve_client, test_case) llmisvc/test_llm_autoscaling_wva.py:892: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd146f1d0> test_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None}, model_name='facebook/opt-125m') def _create_and_wait(kserve_client, test_case): """Create LLMISVC and wait for it to be ready.""" create_llmisvc(kserve_client, test_case.llm_service) > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_autoscaling_wva.py:482: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7f0dd146f1d0>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...e-cle-5a67f5d1'}, {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T17:00:10.807570', start_time = 1785258010.8078423 duration = 900.6285457611084, timestamp_end = '2026-07-28T17:15:11.436391' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f0dd146f1d0> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....toscale-cle-5a67f5d1'}, {'name': 'scaling-hpa-autoscale-cleanup-h-aa1ae037'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7f0dcb7e2de0> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: > raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) E AssertionError: Missing true conditions: {'WorkloadsReady', 'Ready'}, expected {'RouterReady', 'WorkloadsReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-28T17:01:12Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-28T17:01:12Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-28T17:01:12Z', 'severity': 'Info', 'status': 'True', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-28T17:00:47Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-28T17:01:39Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-cleanup-hpa-a41f0e40/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-cleanup-hpa-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-28T17:01:39Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-28T17:01:12Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-cleanup-hpa-a41f0e40/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-cleanup-hpa-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'severity': 'Info', 'status': 'False', 'type': 'ScalingReady'}, {'lastTransitionTime': '2026-07-28T17:01:39Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-28T17:01:12Z', 'message': 'the HPA was unable to compute the replica count: unable to get external metric e2e-test-llm-autoscaling-cleanup-hpa-a41f0e40/wva_desired_replicas/&LabelSelector{MatchLabels:map[string]string{variant_name: autoscale-cleanup-hpa-kserve-hpa,},MatchExpressions:[]LabelSelectorRequirement{},}: unable to fetch metrics from external metrics API: scaledObject name is not specified', 'reason': 'FailedGetExternalMetric', 'status': 'False', 'type': 'WorkloadsReady'}] llmisvc/test_llm_inference_service.py:1371: AssertionErrorself = <e2e.llmisvc.test_llm_canary_lifecycle.TestCanaryLifecycle object at 0x7fc2026dfdd0> test_namespace = 'e2e-test-rollback-b8fdb441' traffic_driver = <function traffic_driver.<locals>.factory at 0x7fc201e4b100> def test_rollback(self, test_namespace, traffic_driver): """Promote v2 then rollback to v1 - traffic returns. (spike step 7)""" api = get_api() ns = test_namespace v1 = MemberSpec(name="roll-v1", weight=9, scheduler=False) v2 = MemberSpec(name="roll-v2", weight=1, scheduler=False) apply_config(api, INFERENCE_SIM.name, ns, INFERENCE_SIM.to_spec()) apply_member(api, v1, ns) apply_member(api, v2, ns) wait_ready(api, v1.name, ns) wait_ready(api, v2.name, ns) gateway = get_gateway_base_url(api, v1.name, ns) driver = traffic_driver( url=f"{gateway}/v1/completions", headers={"X-Gateway-Model-Name": f"publishers/{ns}/models/{MODEL}"}, payload={"model": MODEL, "prompt": "Hello", "max_tokens": 5}, rate=2, timeout=15.0, warmup=True, ) route_headers = {"X-Gateway-Model-Name": f"publishers/{ns}/models/{MODEL}"} route_payload = {"model": MODEL, "prompt": "Hello", "max_tokens": 5} # Promote v2 driver.mark("promote_mutation") patch_weight(api, v1.name, 0, ns) patch_weight(api, v2.name, 9, ns) > wait_for_group_weight(api, v2.name, v1.name, 0, ns) llmisvc/test_llm_canary_lifecycle.py:1216: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ api = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc200e96e90> observer = 'roll-v2', member = 'roll-v1', expected = 0 ns = 'e2e-test-rollback-b8fdb441', timeout = 30 def wait_for_group_weight(api, observer, member, expected, ns, timeout=30): deadline = time.monotonic() + timeout w = None while time.monotonic() < deadline: w = get_group_weight(api, observer, member, ns) if w == expected: return time.sleep(1) > raise TimeoutError(f"{member} weight={w}, expected {expected} (from {observer})") E TimeoutError: roll-v1 weight=None, expected 0 (from roll-v2) llmisvc/test_llm_canary_lifecycle.py:566: TimeoutErrorkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc201cf8f50> name = 'router-with-refs-test' namespace = 'e2e-test-llm-inference-service-028f7809', version = 'v1alpha1' def get_llmisvc( kserve_client: KServeClient, name, namespace, version=constants.KSERVE_V1ALPHA1_VERSION, ): try: > return kserve_client.api_instance.get_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICE, name, ) llmisvc/test_llm_inference_service.py:1204: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc201e1f210> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-028f7809' plural = 'llminferenceservices', name = 'router-with-refs-test' kwargs = {'_return_http_data_only': True} def get_namespaced_custom_object(self, group, version, namespace, plural, name, **kwargs): # noqa: E501 """get_namespaced_custom_object # noqa: E501 Returns a namespace scoped custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.get_namespaced_custom_object(group, version, namespace, plural, name, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: the custom resource's group (required) :param str version: the custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: the custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param str name: the custom object's name (required) :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: object If the method is called asynchronously, returns the request thread. """ kwargs['_return_http_data_only'] = True > return self.get_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, **kwargs) # noqa: E501 ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:1632: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc201e1f210> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-028f7809' plural = 'llminferenceservices', name = 'router-with-refs-test' kwargs = {'_return_http_data_only': True} local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'name', 'async_req', ...], 'auth_settings': ['BearerToken'], 'body_params': None, ...} all_params = ['group', 'version', 'namespace', 'plural', 'name', 'async_req', ...] key = '_return_http_data_only', val = True, collection_formats = {} path_params = {'group': 'serving.kserve.io', 'name': 'router-with-refs-test', 'namespace': 'e2e-test-llm-inference-service-028f7809', 'plural': 'llminferenceservices', ...} query_params = [] def get_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, name, **kwargs): # noqa: E501 """get_namespaced_custom_object # noqa: E501 Returns a namespace scoped custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.get_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: the custom resource's group (required) :param str version: the custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: the custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param str name: the custom object's name (required) :param _return_http_data_only: response data without head status code and headers :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: tuple(object, status_code(int), headers(HTTPHeaderDict)) If the method is called asynchronously, returns the request thread. """ local_var_params = locals() all_params = [ 'group', 'version', 'namespace', 'plural', 'name' ] all_params.extend( [ 'async_req', '_return_http_data_only', '_preload_content', '_request_timeout' ] ) for key, val in six.iteritems(local_var_params['kwargs']): if key not in all_params: raise ApiTypeError( "Got an unexpected keyword argument '%s'" " to method get_namespaced_custom_object" % key ) local_var_params[key] = val del local_var_params['kwargs'] # verify the required parameter 'group' is set if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501 local_var_params['group'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `group` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'version' is set if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501 local_var_params['version'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `version` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'namespace' is set if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501 local_var_params['namespace'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `namespace` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'plural' is set if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501 local_var_params['plural'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `plural` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'name' is set if self.api_client.client_side_validation and ('name' not in local_var_params or # noqa: E501 local_var_params['name'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `name` when calling `get_namespaced_custom_object`") # noqa: E501 collection_formats = {} path_params = {} if 'group' in local_var_params: path_params['group'] = local_var_params['group'] # noqa: E501 if 'version' in local_var_params: path_params['version'] = local_var_params['version'] # noqa: E501 if 'namespace' in local_var_params: path_params['namespace'] = local_var_params['namespace'] # noqa: E501 if 'plural' in local_var_params: path_params['plural'] = local_var_params['plural'] # noqa: E501 if 'name' in local_var_params: path_params['name'] = local_var_params['name'] # noqa: E501 query_params = [] header_params = {} form_params = [] local_var_files = {} body_params = None # HTTP header `Accept` header_params['Accept'] = self.api_client.select_header_accept( ['application/json']) # noqa: E501 # Authentication setting auth_settings = ['BearerToken'] # noqa: E501 > return self.api_client.call_api( '/apis/{group}/{version}/namespaces/{namespace}/{plural}/{name}', 'GET', path_params, query_params, header_params, body=body_params, post_params=form_params, files=local_var_files, response_type='object', # noqa: E501 auth_settings=auth_settings, async_req=local_var_params.get('async_req'), _return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501 _preload_content=local_var_params.get('_preload_content', True), _request_timeout=local_var_params.get('_request_timeout'), collection_formats=collection_formats) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:1739: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc201e1d750> resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}/{name}' method = 'GET' path_params = {'group': 'serving.kserve.io', 'name': 'router-with-refs-test', 'namespace': 'e2e-test-llm-inference-service-028f7809', 'plural': 'llminferenceservices', ...} query_params = [] header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. :param resource_path: Path to method endpoint. :param method: Method to call. :param path_params: Path parameters in the url. :param query_params: Query parameters in the url. :param header_params: Header parameters to be placed in the request header. :param body: Request body. :param post_params dict: Request post form parameters, for `application/x-www-form-urlencoded`, `multipart/form-data`. :param auth_settings list: Auth Settings names for the request. :param response: Response data type. :param files dict: key -> filename, value -> filepath, for `multipart/form-data`. :param async_req bool: execute request asynchronously :param _return_http_data_only: response data without head status code and headers :param collection_formats: dict of collection formats for path, query, header, and post parameters. :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: If async_req parameter is True, the request will be called asynchronously. The method will return the request thread. If parameter async_req is False or missing, then the method will return the response directly. """ if not async_req: > return self.__call_api(resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc201e1d750> resource_path = '/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-028f7809/llminferenceservices/router-with-refs-test' method = 'GET' path_params = [('group', 'serving.kserve.io'), ('version', 'v1alpha1'), ('namespace', 'e2e-test-llm-inference-service-028f7809'), ('plural', 'llminferenceservices'), ('name', 'router-with-refs-test')] query_params = [] header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def __call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict(self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param) ) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = post_params if post_params else [] post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) post_params.extend(self.files_parameters(files)) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url if _host is None: url = self.configuration.host + resource_path else: # use server/host defined in path or operation instead url = _host + resource_path # perform request and return response > response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc201e1d750> method = 'GET' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-028f7809/llminferenceservices/router-with-refs-test' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} post_params = [], body = None, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): """Makes the HTTP request using RESTClient.""" if method == "GET": > return self.rest_client.GET(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:373: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc201e1e790> url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-028f7809/llminferenceservices/router-with-refs-test' headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} query_params = [], _preload_content = True, _request_timeout = None def GET(self, url, headers=None, query_params=None, _preload_content=True, _request_timeout=None): > return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, query_params=query_params) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:244: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc201e1e790> method = 'GET' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-028f7809/llminferenceservices/router-with-refs-test' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = {}, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Perform requests. :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] if post_params and body: raise ApiValueError( "body parameter cannot be used with post_params parameter." ) post_params = post_params or {} headers = headers or {} timeout = None if _request_timeout: if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821 timeout = urllib3.Timeout(total=_request_timeout) elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): timeout = urllib3.Timeout( connect=_request_timeout[0], read=_request_timeout[1]) if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) if (re.search('json', headers['Content-Type'], re.IGNORECASE) or headers['Content-Type'] == 'application/apply-patch+yaml'): if headers['Content-Type'] == 'application/json-patch+json': if not isinstance(body, list): headers['Content-Type'] = \ 'application/strategic-merge-patch+json' request_body = None if body is not None: request_body = json.dumps(body) r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=False, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by urllib3 will be # overwritten. del headers['Content-Type'] r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=True, preload_content=_preload_content, timeout=timeout, headers=headers) # Pass a `string` parameter directly in the body to support # other content types than Json when `body` argument is # provided in serialized form elif isinstance(body, str) or isinstance(body, bytes): request_body = body r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) # For `GET`, `HEAD` else: r = self.pool_manager.request(method, url, fields=query_params, preload_content=_preload_content, timeout=timeout, headers=headers) except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) # In the python 3, the response.data is bytes. # we need to decode it to string. if six.PY3: r.data = r.data.decode('utf8') # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: > raise ApiException(http_resp=r) E kubernetes.client.exceptions.ApiException: (500) E Reason: Internal Server Error E HTTP response headers: HTTPHeaderDict({'Audit-Id': '8d52e004-35ea-4781-b7af-47e2b20be695', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': 'ff0b8d77-f0bc-468d-a86a-0524cad8e456', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'a7ead034-f650-4220-8a0e-0379eadce096', 'Date': 'Tue, 28 Jul 2026 17:24:04 GMT', 'Content-Length': '264'}) E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"conversion webhook for serving.kserve.io/v1alpha2, Kind=LLMInferenceService failed: Post \"https://llmisvc-webhook-server-service.kserve.svc:443/convert?timeout=30s\": EOF","code":500} ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException The above exception was the direct cause of the following exception: test_case = TestCase(base_refs=['router-with-refs', 'scheduler-managed', 'workload-single-cpu', 'model-fb-opt-125m'], prompt='KSer... {'name': 'model-fb-opt-125m-router-with-r-6d64416a'}]}, 'status': None}, model_name='facebook/opt-125m') @pytest.mark.asyncio(loop_scope="session") @pytest.mark.parametrize( "test_case", [ pytest.param( TestCase( base_refs=[ "router-with-gateway-ref", "router-with-managed-route", "model-fb-opt-125m", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=create_response_assertion(with_field="choices"), expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="custom-route-timeout-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs", "scheduler-managed", "workload-single-cpu", "model-fb-opt-125m", ], prompt="KServe is a", service_name="router-with-refs-test", expected_gateway="router-gateway-1", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-1", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-1", tc.namespace, "router-gateway-1", "router-with-refs-test", ), make_router_health_route( "router-route-2", tc.namespace, "router-gateway-1", "router-with-refs-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=["router-managed", "workload-pd-cpu", "model-fb-opt-125m"], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-custom-route-timeout-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="custom-route-timeout-pd-test", response_assertion=assert_200_with_choices, ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-with-refs-pd", "scheduler-managed", "workload-pd-cpu", "model-fb-opt-125m", ], prompt="You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. " "Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. " "Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.", service_name="router-with-refs-pd-test", response_assertion=assert_200_with_choices, expected_gateway="router-gateway-2", before_test=[ lambda tc: create_router_resources( gateways=[ make_router_gateway( "router-gateway-2", tc.namespace, ), ], routes=[ make_router_main_route( "router-route-3", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), make_router_health_route( "router-route-4", tc.namespace, "router-gateway-2", "router-with-refs-pd-test", ), ], ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.custom_gateway, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-dp-ep-gpu", "workload-dp-ep-prefill-gpu", "model-deepseek-v2-lite", ], prompt="Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically " "where the compute plane (P) and the data plane (D) are independently deployed and managed for a " "geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the " "fundamental challenges of network latency and data consistency, elaborate on the advanced " "considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: " "How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to " "evolve to support optimal performance and minimize inter-plane communication overhead, especially for " "synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically " "optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: " "Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) " "and their applicability in balancing performance and data integrity across a globally distributed data plane. " "Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, " "intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. " "3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently " "manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, " "cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). " "Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on " "workload patterns and data locality, potentially involving live migration strategies. " "4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter " "challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), " "fine-grained access control to data at rest and in motion, and identity management across disaggregated " "components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) " "concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: " "Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and " "data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) " "would be essential? How would incident response and troubleshooting differ in this disaggregated environment " "compared to traditional integrated systems? Consider the challenges of pinpointing root causes across " "independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries " "or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) " "where the benefits of P/D disaggregation would strongly outweigh its complexities. " "Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions " "directly interacting with object storage, in-memory disaggregation) that could further drive or " "transform P/D disaggregation in cloud computing.", max_tokens=2000, ), marks=[ pytest.mark.cluster_gpu, pytest.mark.cluster_nvidia, pytest.mark.cluster_nvidia_roce, ], ), pytest.param( TestCase( base_refs=[ "router-no-scheduler", "workload-single-cpu", "model-fb-opt-125m", ], prompt="What is KServe?", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.no_scheduler, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-fb-opt-125m", ], prompt="This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, " "but without the resources requirements for DP+EP (GPUs and ROCe/IB).", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node], ), # Scheduler config tests pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-inline-config", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-inline-config-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Chat completions endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], model_name="Qwen/Qwen2.5-0.5B-Instruct", endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=create_response_assertion(with_field="choices"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-configmap-ref", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-configmap-ref-test", before_test=[ lambda tc: create_scheduler_configmap(namespace=tc.namespace) ], after_test=[ lambda tc: delete_scheduler_configmap(namespace=tc.namespace) ], ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-replicas", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-ha-replicas-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-custom-template", "workload-llmd-simulator", ], prompt="KServe is a", service_name="scheduler-custom-template-test", ), marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node], ), # Scheduler v0.6 → v0.7 migration tests. # Deploy v0.6-style configs and verify the controller migrates them # so the v0.7 scheduler boots successfully. pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-pd-config-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-pd-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-v06-nonzero-threshold-migration", "workload-llmd-simulator-pd", ], prompt="KServe is a", service_name="scheduler-v06-threshold-migration-test", response_assertion=assert_200_with_choices, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — clean path: token-producer in inline config pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-tokenizer-kvcache", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-clean-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Standalone tokenizer — migration path: legacy precise-prefix-cache-scorer # triggers auto-provisioned tokenizer without explicit tokenizer:{} field pytest.param( TestCase( base_refs=[ "router-managed", "scheduler-with-precise-prefix-cache-inline-config", "workload-llmd-simulator-kvcache", ], prompt="KServe is a", service_name="tokenizer-migration-path-test", ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Models endpoint coverage pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/models", response_assertion=create_response_assertion(with_field="data"), ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/completions", prompt="KServe is a", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/chat/completions pytest.param( TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches("facebook/opt-125m"), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, peers=[ TestCase( base_refs=[ "router-managed", "workload-llmd-simulator", "model-qwen2.5-0.5b", ], endpoint="/v1/chat/completions", prompt="What is KServe?", payload_formatter=chat_completions_payload, response_assertion=assert_model_field_matches( "Qwen/Qwen2.5-0.5B-Instruct" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/Qwen/Qwen2.5-0.5B-Instruct", }, ), ], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator, pytest.mark.model_routing, ], ), # Model-based routing via X-Gateway-Model-Name header — LoRA adapter pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/completions", prompt="KServe is a", model_name="publishers/{namespace}/models/lora-adapter-1", payload_formatter=completions_payload, response_assertion=assert_model_field_matches( "publishers/{namespace}/models/lora-adapter-1" ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/lora-adapter-1", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # Model-based routing via X-Gateway-Model-Name header — /v1/models (base + LoRA) pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-fb-opt-125m-with-lora-hf", ], endpoint="/v1/models", response_assertion=assert_models_contains( "facebook/opt-125m", "publishers/{namespace}/models/facebook/opt-125m", "lora-adapter-1", "publishers/{namespace}/models/lora-adapter-1", ), url_getter=get_model_routing_url, extra_headers={ MODEL_ROUTING_HEADER: "publishers/{namespace}/models/facebook/opt-125m", }, ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.model_routing, pytest.mark.lora, ], ), # PVC storage tests -- validate direct PVC volume mount with real vLLM serving pytest.param( TestCase( base_refs=[ "router-managed", "workload-single-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-pd-cpu", "model-pvc", ], prompt="KServe is a", response_assertion=assert_200_with_choices, before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.pvc_storage, ], ), pytest.param( TestCase( base_refs=[ "router-managed", "workload-simulated-dp-ep-cpu", "model-pvc", ], prompt="KServe is a", before_test=[lambda tc: ensure_pvc_with_model(namespace=tc.namespace)], ), marks=[ pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node, pytest.mark.pvc_storage, ], ), ], indirect=["test_case"], ids=generate_test_id, ) @log_execution def test_llm_inference_service(test_case: TestCase): # noqa: F811 inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) service_name = test_case.llm_service.metadata.name prefix = test_case.log_prefix test_failed = False try: print(f"{prefix} Creating LLMInferenceService {service_name}") create_llmisvc(kserve_client, test_case.llm_service) print(f"{prefix} Waiting for LLMInferenceService {service_name} to be ready") > wait_for_llm_isvc_ready( kserve_client, test_case.llm_service, test_case.wait_timeout ) llmisvc/test_llm_inference_service.py:866: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = (<kserve.api.kserve_client.KServeClient object at 0x7fc201cf8f50>, {'api_version': 'serving.kserve.io/v1alpha1', 'kin...-with-ec5d4bfa'}, {'name': 'model-fb-opt-125m-router-with-r-6d64416a'}]}, 'status': None}, 900) kwargs = {}, func_name = 'wait_for_llm_isvc_ready' timestamp_start = '2026-07-28T17:22:22.706303', start_time = 1785259342.706591 duration = 102.21241092681885, timestamp_end = '2026-07-28T17:24:04.919006' @functools.wraps(func) def wrapper(*args, **kwargs): func_name = func.__name__ timestamp_start = datetime.now().isoformat() logger.info( f"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}" ) start_time = time.time() try: > result = func(*args, **kwargs) llmisvc/logging.py:40: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc201cf8f50> given = {'api_version': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'annotations': {'security....router-with-ec5d4bfa'}, {'name': 'model-fb-opt-125m-router-with-r-6d64416a'}]}, 'status': None} timeout_seconds = 900 @log_execution def wait_for_llm_isvc_ready( kserve_client: KServeClient, given: V1alpha1LLMInferenceService, timeout_seconds: int = 900, ) -> str: def assert_llm_isvc_ready(): out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) if "status" not in out: raise AssertionError("No status found in LLM inference service") status = out["status"] if "conditions" not in status: raise AssertionError("No conditions found in status") expected_true_conditions = {"Ready", "WorkloadsReady", "RouterReady"} got_true_conditions = set() all_condition_types = set() conditions = status["conditions"] for condition in conditions: ctype = condition.get("type") all_condition_types.add(ctype) if condition.get("status") == "True": got_true_conditions.add(ctype) # When TokenizerReady is present, it must also be True if "TokenizerReady" in all_condition_types: expected_true_conditions.add("TokenizerReady") missing_conditions = expected_true_conditions - got_true_conditions if missing_conditions: raise AssertionError( f"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}" ) return True > return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0) llmisvc/test_llm_inference_service.py:1376: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ assertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7fc2021b6020> timeout = 900, interval = 1.0 def wait_for( assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1 ) -> Any: """Wait for the assertion to succeed within timeout.""" deadline = time.time() + timeout last_msg = None while True: try: > return assertion_fn() llmisvc/test_llm_inference_service.py:1387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def assert_llm_isvc_ready(): > out = get_llmisvc( kserve_client, given.metadata.name, given.metadata.namespace, given.api_version.split("/")[1], ) llmisvc/test_llm_inference_service.py:1339: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc201cf8f50> name = 'router-with-refs-test' namespace = 'e2e-test-llm-inference-service-028f7809', version = 'v1alpha1' def get_llmisvc( kserve_client: KServeClient, name, namespace, version=constants.KSERVE_V1ALPHA1_VERSION, ): try: return kserve_client.api_instance.get_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICE, name, ) except client.rest.ApiException as e: > raise RuntimeError( f"❌ Exception when calling CustomObjectsApi->" f"get_namespaced_custom_object for LLMInferenceService: {e}" ) from e E RuntimeError: ❌ Exception when calling CustomObjectsApi->get_namespaced_custom_object for LLMInferenceService: (500) E Reason: Internal Server Error E HTTP response headers: HTTPHeaderDict({'Audit-Id': '8d52e004-35ea-4781-b7af-47e2b20be695', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': 'ff0b8d77-f0bc-468d-a86a-0524cad8e456', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'a7ead034-f650-4220-8a0e-0379eadce096', 'Date': 'Tue, 28 Jul 2026 17:24:04 GMT', 'Content-Length': '264'}) E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"conversion webhook for serving.kserve.io/v1alpha2, Kind=LLMInferenceService failed: Post \"https://llmisvc-webhook-server-service.kserve.svc:443/convert?timeout=30s\": EOF","code":500} llmisvc/test_llm_inference_service.py:1212: RuntimeErrorkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc202413a90> llm_config = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} namespace = 'e2e-test-llm-inference-service-62876e81' def _create_or_update_llmisvc_config(kserve_client, llm_config, namespace=None): """Create or update an LLMInferenceServiceConfig resource.""" version = llm_config["apiVersion"].split("/")[1] if namespace is None: namespace = llm_config.get("metadata", {}).get("namespace", "default") name = llm_config.get("metadata", {}).get("name") if not name: raise ValueError("LLMInferenceServiceConfig must have a name in metadata") logger.info(f"Checking LLMInferenceServiceConfig {name} in namespace {namespace}") try: > existing_config = kserve_client.api_instance.get_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, name, ) llmisvc/fixtures.py:1721: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202144590> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-62876e81' plural = 'llminferenceserviceconfigs' name = 'router-managed-llmisvc-model-fb-b69cdb68' kwargs = {'_return_http_data_only': True} def get_namespaced_custom_object(self, group, version, namespace, plural, name, **kwargs): # noqa: E501 """get_namespaced_custom_object # noqa: E501 Returns a namespace scoped custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.get_namespaced_custom_object(group, version, namespace, plural, name, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: the custom resource's group (required) :param str version: the custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: the custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param str name: the custom object's name (required) :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: object If the method is called asynchronously, returns the request thread. """ kwargs['_return_http_data_only'] = True > return self.get_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, **kwargs) # noqa: E501 ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:1632: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202144590> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-62876e81' plural = 'llminferenceserviceconfigs' name = 'router-managed-llmisvc-model-fb-b69cdb68' kwargs = {'_return_http_data_only': True} local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'name', 'async_req', ...], 'auth_settings': ['BearerToken'], 'body_params': None, ...} all_params = ['group', 'version', 'namespace', 'plural', 'name', 'async_req', ...] key = '_return_http_data_only', val = True, collection_formats = {} path_params = {'group': 'serving.kserve.io', 'name': 'router-managed-llmisvc-model-fb-b69cdb68', 'namespace': 'e2e-test-llm-inference-service-62876e81', 'plural': 'llminferenceserviceconfigs', ...} query_params = [] def get_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, name, **kwargs): # noqa: E501 """get_namespaced_custom_object # noqa: E501 Returns a namespace scoped custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.get_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: the custom resource's group (required) :param str version: the custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: the custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param str name: the custom object's name (required) :param _return_http_data_only: response data without head status code and headers :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: tuple(object, status_code(int), headers(HTTPHeaderDict)) If the method is called asynchronously, returns the request thread. """ local_var_params = locals() all_params = [ 'group', 'version', 'namespace', 'plural', 'name' ] all_params.extend( [ 'async_req', '_return_http_data_only', '_preload_content', '_request_timeout' ] ) for key, val in six.iteritems(local_var_params['kwargs']): if key not in all_params: raise ApiTypeError( "Got an unexpected keyword argument '%s'" " to method get_namespaced_custom_object" % key ) local_var_params[key] = val del local_var_params['kwargs'] # verify the required parameter 'group' is set if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501 local_var_params['group'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `group` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'version' is set if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501 local_var_params['version'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `version` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'namespace' is set if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501 local_var_params['namespace'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `namespace` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'plural' is set if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501 local_var_params['plural'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `plural` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'name' is set if self.api_client.client_side_validation and ('name' not in local_var_params or # noqa: E501 local_var_params['name'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `name` when calling `get_namespaced_custom_object`") # noqa: E501 collection_formats = {} path_params = {} if 'group' in local_var_params: path_params['group'] = local_var_params['group'] # noqa: E501 if 'version' in local_var_params: path_params['version'] = local_var_params['version'] # noqa: E501 if 'namespace' in local_var_params: path_params['namespace'] = local_var_params['namespace'] # noqa: E501 if 'plural' in local_var_params: path_params['plural'] = local_var_params['plural'] # noqa: E501 if 'name' in local_var_params: path_params['name'] = local_var_params['name'] # noqa: E501 query_params = [] header_params = {} form_params = [] local_var_files = {} body_params = None # HTTP header `Accept` header_params['Accept'] = self.api_client.select_header_accept( ['application/json']) # noqa: E501 # Authentication setting auth_settings = ['BearerToken'] # noqa: E501 > return self.api_client.call_api( '/apis/{group}/{version}/namespaces/{namespace}/{plural}/{name}', 'GET', path_params, query_params, header_params, body=body_params, post_params=form_params, files=local_var_files, response_type='object', # noqa: E501 auth_settings=auth_settings, async_req=local_var_params.get('async_req'), _return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501 _preload_content=local_var_params.get('_preload_content', True), _request_timeout=local_var_params.get('_request_timeout'), collection_formats=collection_formats) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:1739: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202144d90> resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}/{name}' method = 'GET' path_params = {'group': 'serving.kserve.io', 'name': 'router-managed-llmisvc-model-fb-b69cdb68', 'namespace': 'e2e-test-llm-inference-service-62876e81', 'plural': 'llminferenceserviceconfigs', ...} query_params = [] header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. :param resource_path: Path to method endpoint. :param method: Method to call. :param path_params: Path parameters in the url. :param query_params: Query parameters in the url. :param header_params: Header parameters to be placed in the request header. :param body: Request body. :param post_params dict: Request post form parameters, for `application/x-www-form-urlencoded`, `multipart/form-data`. :param auth_settings list: Auth Settings names for the request. :param response: Response data type. :param files dict: key -> filename, value -> filepath, for `multipart/form-data`. :param async_req bool: execute request asynchronously :param _return_http_data_only: response data without head status code and headers :param collection_formats: dict of collection formats for path, query, header, and post parameters. :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: If async_req parameter is True, the request will be called asynchronously. The method will return the request thread. If parameter async_req is False or missing, then the method will return the response directly. """ if not async_req: > return self.__call_api(resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202144d90> resource_path = '/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs/router-managed-llmisvc-model-fb-b69cdb68' method = 'GET' path_params = [('group', 'serving.kserve.io'), ('version', 'v1alpha1'), ('namespace', 'e2e-test-llm-inference-service-62876e81'), ('plural', 'llminferenceserviceconfigs'), ('name', 'router-managed-llmisvc-model-fb-b69cdb68')] query_params = [] header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def __call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict(self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param) ) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = post_params if post_params else [] post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) post_params.extend(self.files_parameters(files)) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url if _host is None: url = self.configuration.host + resource_path else: # use server/host defined in path or operation instead url = _host + resource_path # perform request and return response > response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202144d90> method = 'GET' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1a...namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs/router-managed-llmisvc-model-fb-b69cdb68' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} post_params = [], body = None, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): """Makes the HTTP request using RESTClient.""" if method == "GET": > return self.rest_client.GET(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:373: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202147a50> url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1a...namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs/router-managed-llmisvc-model-fb-b69cdb68' headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} query_params = [], _preload_content = True, _request_timeout = None def GET(self, url, headers=None, query_params=None, _preload_content=True, _request_timeout=None): > return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, query_params=query_params) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:244: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202147a50> method = 'GET' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1a...namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs/router-managed-llmisvc-model-fb-b69cdb68' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = {}, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Perform requests. :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] if post_params and body: raise ApiValueError( "body parameter cannot be used with post_params parameter." ) post_params = post_params or {} headers = headers or {} timeout = None if _request_timeout: if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821 timeout = urllib3.Timeout(total=_request_timeout) elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): timeout = urllib3.Timeout( connect=_request_timeout[0], read=_request_timeout[1]) if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) if (re.search('json', headers['Content-Type'], re.IGNORECASE) or headers['Content-Type'] == 'application/apply-patch+yaml'): if headers['Content-Type'] == 'application/json-patch+json': if not isinstance(body, list): headers['Content-Type'] = \ 'application/strategic-merge-patch+json' request_body = None if body is not None: request_body = json.dumps(body) r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=False, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by urllib3 will be # overwritten. del headers['Content-Type'] r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=True, preload_content=_preload_content, timeout=timeout, headers=headers) # Pass a `string` parameter directly in the body to support # other content types than Json when `body` argument is # provided in serialized form elif isinstance(body, str) or isinstance(body, bytes): request_body = body r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) # For `GET`, `HEAD` else: r = self.pool_manager.request(method, url, fields=query_params, preload_content=_preload_content, timeout=timeout, headers=headers) except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) # In the python 3, the response.data is bytes. # we need to decode it to string. if six.PY3: r.data = r.data.decode('utf8') # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: > raise ApiException(http_resp=r) E kubernetes.client.exceptions.ApiException: (404) E Reason: Not Found E HTTP response headers: HTTPHeaderDict({'Audit-Id': '1c81504e-9421-406f-9182-f2e84d2c5efd', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': 'ff0b8d77-f0bc-468d-a86a-0524cad8e456', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'a7ead034-f650-4220-8a0e-0379eadce096', 'Date': 'Tue, 28 Jul 2026 17:24:06 GMT', 'Content-Length': '338'}) E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"llminferenceserviceconfigs.serving.kserve.io \"router-managed-llmisvc-model-fb-b69cdb68\" not found","reason":"NotFound","details":{"name":"router-managed-llmisvc-model-fb-b69cdb68","group":"serving.kserve.io","kind":"llminferenceserviceconfigs"},"code":404} ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException During handling of the above exception, another exception occurred: request = <SubRequest 'ensure_gateway_proxy_memory' for <Function test_llm_inference_service[router-managed-workload-pd-cpu-model-fb-opt-125m]>> @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(request): """After test setup creates gateways, patch them for proxy memory.""" if not GATEWAY_PROXY_MEMORY: return # Let test_case (llmisvc) create gateways first if "test_case" in request.fixturenames: > request.getfixturevalue("test_case") common/gateway_proxy_istio.py:183: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ request = <SubRequest 'test_case' for <Function test_llm_inference_service[router-managed-workload-pd-cpu-model-fb-opt-125m]>> test_namespace = 'e2e-test-llm-inference-service-62876e81' @pytest.fixture(scope="function") def test_case(request, test_namespace): tc = request.param ns = test_namespace inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) tc.namespace = ns for peer in tc.peers: peer.namespace = ns for func in tc.before_test: func(tc) > _setup_test_case_service(kserve_client, tc, request.node.name, namespace=ns) llmisvc/fixtures.py:1611: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc202413a90> tc = TestCase(base_refs=['router-managed', 'workload-pd-cpu', 'model-fb-opt-125m'], prompt='You are an expert in Kubernetes...inference-service-62876e81', before_test=[], after_test=[], peers=[], llm_service=None, model_name='facebook/opt-125m') test_node_name = 'test_llm_inference_service[router-managed-workload-pd-cpu-model-fb-opt-125m]' namespace = 'e2e-test-llm-inference-service-62876e81', peer_index = None def _setup_test_case_service( kserve_client, tc, test_node_name, namespace, peer_index=None ): """Create LLMInferenceServiceConfigs and build the LLMInferenceService for a TestCase. Returns a list of created config names for cleanup tracking. """ missing_refs = [ ref for ref in tc.base_refs if ref not in LLMINFERENCESERVICE_CONFIGS ] if missing_refs: raise ValueError( f"Missing base_refs in LLMINFERENCESERVICE_CONFIGS: {missing_refs}" ) if not tc.service_name: suffix = f"-peer-{peer_index}" if peer_index is not None else "" tc.service_name = generate_service_name(test_node_name + suffix, tc.base_refs) if tc.model_name == "default/model": tc.model_name = _get_model_name_from_configs(tc.base_refs) elif "{namespace}" in tc.model_name: tc.model_name = tc.model_name.format(namespace=namespace) created_configs = [] unique_base_refs = [] for base_ref in tc.base_refs: unique_config_name = generate_k8s_safe_suffix(base_ref, [tc.service_name]) unique_base_refs.append(unique_config_name) config = LLMINFERENCESERVICE_CONFIGS[base_ref] spec = config(namespace) if callable(config) else copy.deepcopy(config) unique_config_body = { "apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceServiceConfig", "metadata": { "name": unique_config_name, "namespace": namespace, }, "spec": spec, } > _create_or_update_llmisvc_config(kserve_client, unique_config_body, namespace) llmisvc/fixtures.py:1573: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc202413a90> llm_config = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} namespace = 'e2e-test-llm-inference-service-62876e81' def _create_or_update_llmisvc_config(kserve_client, llm_config, namespace=None): """Create or update an LLMInferenceServiceConfig resource.""" version = llm_config["apiVersion"].split("/")[1] if namespace is None: namespace = llm_config.get("metadata", {}).get("namespace", "default") name = llm_config.get("metadata", {}).get("name") if not name: raise ValueError("LLMInferenceServiceConfig must have a name in metadata") logger.info(f"Checking LLMInferenceServiceConfig {name} in namespace {namespace}") try: existing_config = kserve_client.api_instance.get_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, name, ) llm_config["metadata"] = existing_config["metadata"] outputs = kserve_client.api_instance.replace_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, name, llm_config, ) logger.info(f"✓ Successfully updated LLMInferenceServiceConfig {name}") return outputs except client.rest.ApiException as e: if e.status == 404: # Not found - create it logger.info( f"Resource not found, creating LLMInferenceServiceConfig {name}" ) > outputs = kserve_client.api_instance.create_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, llm_config, ) llmisvc/fixtures.py:1747: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202144590> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-62876e81' plural = 'llminferenceserviceconfigs' body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} kwargs = {'_return_http_data_only': True} def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501 """create_namespaced_custom_object # noqa: E501 Creates a namespace scoped Custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: The custom resource's group name (required) :param str version: The custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param object body: The JSON schema of the Resource to create. (required) :param str pretty: If 'true', then the output is pretty printed. :param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed :param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. :param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional) :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: object If the method is called asynchronously, returns the request thread. """ kwargs['_return_http_data_only'] = True > return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501 ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202144590> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-62876e81' plural = 'llminferenceserviceconfigs' body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} kwargs = {'_return_http_data_only': True} local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au...e-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [...]}}}}}, ...} all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...] key = '_return_http_data_only', val = True, collection_formats = {} path_params = {'group': 'serving.kserve.io', 'namespace': 'e2e-test-llm-inference-service-62876e81', 'plural': 'llminferenceserviceconfigs', 'version': 'v1alpha1'} query_params = [] def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501 """create_namespaced_custom_object # noqa: E501 Creates a namespace scoped Custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: The custom resource's group name (required) :param str version: The custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param object body: The JSON schema of the Resource to create. (required) :param str pretty: If 'true', then the output is pretty printed. :param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed :param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. :param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional) :param _return_http_data_only: response data without head status code and headers :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: tuple(object, status_code(int), headers(HTTPHeaderDict)) If the method is called asynchronously, returns the request thread. """ local_var_params = locals() all_params = [ 'group', 'version', 'namespace', 'plural', 'body', 'pretty', 'dry_run', 'field_manager', 'field_validation' ] all_params.extend( [ 'async_req', '_return_http_data_only', '_preload_content', '_request_timeout' ] ) for key, val in six.iteritems(local_var_params['kwargs']): if key not in all_params: raise ApiTypeError( "Got an unexpected keyword argument '%s'" " to method create_namespaced_custom_object" % key ) local_var_params[key] = val del local_var_params['kwargs'] # verify the required parameter 'group' is set if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501 local_var_params['group'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'version' is set if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501 local_var_params['version'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'namespace' is set if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501 local_var_params['namespace'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'plural' is set if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501 local_var_params['plural'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'body' is set if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501 local_var_params['body'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501 collection_formats = {} path_params = {} if 'group' in local_var_params: path_params['group'] = local_var_params['group'] # noqa: E501 if 'version' in local_var_params: path_params['version'] = local_var_params['version'] # noqa: E501 if 'namespace' in local_var_params: path_params['namespace'] = local_var_params['namespace'] # noqa: E501 if 'plural' in local_var_params: path_params['plural'] = local_var_params['plural'] # noqa: E501 query_params = [] if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501 query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501 if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501 query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501 if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501 query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501 if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501 query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501 header_params = {} form_params = [] local_var_files = {} body_params = None if 'body' in local_var_params: body_params = local_var_params['body'] # HTTP header `Accept` header_params['Accept'] = self.api_client.select_header_accept( ['application/json']) # noqa: E501 # Authentication setting auth_settings = ['BearerToken'] # noqa: E501 > return self.api_client.call_api( '/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST', path_params, query_params, header_params, body=body_params, post_params=form_params, files=local_var_files, response_type='object', # noqa: E501 auth_settings=auth_settings, async_req=local_var_params.get('async_req'), _return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501 _preload_content=local_var_params.get('_preload_content', True), _request_timeout=local_var_params.get('_request_timeout'), collection_formats=collection_formats) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202144d90> resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}' method = 'POST' path_params = {'group': 'serving.kserve.io', 'namespace': 'e2e-test-llm-inference-service-62876e81', 'plural': 'llminferenceserviceconfigs', 'version': 'v1alpha1'} query_params = [] header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. :param resource_path: Path to method endpoint. :param method: Method to call. :param path_params: Path parameters in the url. :param query_params: Query parameters in the url. :param header_params: Header parameters to be placed in the request header. :param body: Request body. :param post_params dict: Request post form parameters, for `application/x-www-form-urlencoded`, `multipart/form-data`. :param auth_settings list: Auth Settings names for the request. :param response: Response data type. :param files dict: key -> filename, value -> filepath, for `multipart/form-data`. :param async_req bool: execute request asynchronously :param _return_http_data_only: response data without head status code and headers :param collection_formats: dict of collection formats for path, query, header, and post parameters. :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: If async_req parameter is True, the request will be called asynchronously. The method will return the request thread. If parameter async_req is False or missing, then the method will return the response directly. """ if not async_req: > return self.__call_api(resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202144d90> resource_path = '/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs' method = 'POST' path_params = [('group', 'serving.kserve.io'), ('version', 'v1alpha1'), ('namespace', 'e2e-test-llm-inference-service-62876e81'), ('plural', 'llminferenceserviceconfigs')] query_params = [] header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def __call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict(self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param) ) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = post_params if post_params else [] post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) post_params.extend(self.files_parameters(files)) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url if _host is None: url = self.configuration.host + resource_path else: # use server/host defined in path or operation instead url = _host + resource_path # perform request and return response > response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202144d90> method = 'POST' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} post_params = [] body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.GET(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, headers=headers) elif method == "HEAD": return self.rest_client.HEAD(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, headers=headers) elif method == "OPTIONS": return self.rest_client.OPTIONS(url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout) elif method == "POST": > return self.rest_client.POST(url, query_params=query_params, headers=headers, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202147a50> url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs' headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} query_params = [], post_params = [] body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} _preload_content = True, _request_timeout = None def POST(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): > return self.request("POST", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202147a50> method = 'POST' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-62876e81/llminferenceserviceconfigs' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-managed...rvice-62876e81'}, 'spec': {'router': {'gateway': {}, 'route': {}, 'scheduler': {'template': {'containers': [{...}]}}}}} post_params = {}, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Perform requests. :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] if post_params and body: raise ApiValueError( "body parameter cannot be used with post_params parameter." ) post_params = post_params or {} headers = headers or {} timeout = None if _request_timeout: if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821 timeout = urllib3.Timeout(total=_request_timeout) elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): timeout = urllib3.Timeout( connect=_request_timeout[0], read=_request_timeout[1]) if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) if (re.search('json', headers['Content-Type'], re.IGNORECASE) or headers['Content-Type'] == 'application/apply-patch+yaml'): if headers['Content-Type'] == 'application/json-patch+json': if not isinstance(body, list): headers['Content-Type'] = \ 'application/strategic-merge-patch+json' request_body = None if body is not None: request_body = json.dumps(body) r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=False, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by urllib3 will be # overwritten. del headers['Content-Type'] r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=True, preload_content=_preload_content, timeout=timeout, headers=headers) # Pass a `string` parameter directly in the body to support # other content types than Json when `body` argument is # provided in serialized form elif isinstance(body, str) or isinstance(body, bytes): request_body = body r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) # For `GET`, `HEAD` else: r = self.pool_manager.request(method, url, fields=query_params, preload_content=_preload_content, timeout=timeout, headers=headers) except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) # In the python 3, the response.data is bytes. # we need to decode it to string. if six.PY3: r.data = r.data.decode('utf8') # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: > raise ApiException(http_resp=r) E kubernetes.client.exceptions.ApiException: (500) E Reason: Internal Server Error E HTTP response headers: HTTPHeaderDict({'Audit-Id': '4b3f3cac-d2e3-4b14-beb8-d3bea22125be', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': 'ff0b8d77-f0bc-468d-a86a-0524cad8e456', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'a7ead034-f650-4220-8a0e-0379eadce096', 'Date': 'Tue, 28 Jul 2026 17:24:06 GMT', 'Content-Length': '701'}) E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.v1alpha1.validator\": failed to call webhook: Post \"https://llmisvc-webhook-server-service.kserve.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": EOF","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.v1alpha1.validator\": failed to call webhook: Post \"https://llmisvc-webhook-server-service.kserve.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": EOF"}]},"code":500} ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiExceptionkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc201e70610> llm_config = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} namespace = 'e2e-test-llm-inference-service-d73d44f4' def _create_or_update_llmisvc_config(kserve_client, llm_config, namespace=None): """Create or update an LLMInferenceServiceConfig resource.""" version = llm_config["apiVersion"].split("/")[1] if namespace is None: namespace = llm_config.get("metadata", {}).get("namespace", "default") name = llm_config.get("metadata", {}).get("name") if not name: raise ValueError("LLMInferenceServiceConfig must have a name in metadata") logger.info(f"Checking LLMInferenceServiceConfig {name} in namespace {namespace}") try: > existing_config = kserve_client.api_instance.get_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, name, ) llmisvc/fixtures.py:1721: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202133e10> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-d73d44f4' plural = 'llminferenceserviceconfigs' name = 'router-custom-route-timeout-pd-8b408578' kwargs = {'_return_http_data_only': True} def get_namespaced_custom_object(self, group, version, namespace, plural, name, **kwargs): # noqa: E501 """get_namespaced_custom_object # noqa: E501 Returns a namespace scoped custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.get_namespaced_custom_object(group, version, namespace, plural, name, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: the custom resource's group (required) :param str version: the custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: the custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param str name: the custom object's name (required) :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: object If the method is called asynchronously, returns the request thread. """ kwargs['_return_http_data_only'] = True > return self.get_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, **kwargs) # noqa: E501 ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:1632: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202133e10> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-d73d44f4' plural = 'llminferenceserviceconfigs' name = 'router-custom-route-timeout-pd-8b408578' kwargs = {'_return_http_data_only': True} local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'name', 'async_req', ...], 'auth_settings': ['BearerToken'], 'body_params': None, ...} all_params = ['group', 'version', 'namespace', 'plural', 'name', 'async_req', ...] key = '_return_http_data_only', val = True, collection_formats = {} path_params = {'group': 'serving.kserve.io', 'name': 'router-custom-route-timeout-pd-8b408578', 'namespace': 'e2e-test-llm-inference-service-d73d44f4', 'plural': 'llminferenceserviceconfigs', ...} query_params = [] def get_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, name, **kwargs): # noqa: E501 """get_namespaced_custom_object # noqa: E501 Returns a namespace scoped custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.get_namespaced_custom_object_with_http_info(group, version, namespace, plural, name, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: the custom resource's group (required) :param str version: the custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: the custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param str name: the custom object's name (required) :param _return_http_data_only: response data without head status code and headers :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: tuple(object, status_code(int), headers(HTTPHeaderDict)) If the method is called asynchronously, returns the request thread. """ local_var_params = locals() all_params = [ 'group', 'version', 'namespace', 'plural', 'name' ] all_params.extend( [ 'async_req', '_return_http_data_only', '_preload_content', '_request_timeout' ] ) for key, val in six.iteritems(local_var_params['kwargs']): if key not in all_params: raise ApiTypeError( "Got an unexpected keyword argument '%s'" " to method get_namespaced_custom_object" % key ) local_var_params[key] = val del local_var_params['kwargs'] # verify the required parameter 'group' is set if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501 local_var_params['group'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `group` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'version' is set if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501 local_var_params['version'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `version` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'namespace' is set if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501 local_var_params['namespace'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `namespace` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'plural' is set if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501 local_var_params['plural'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `plural` when calling `get_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'name' is set if self.api_client.client_side_validation and ('name' not in local_var_params or # noqa: E501 local_var_params['name'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `name` when calling `get_namespaced_custom_object`") # noqa: E501 collection_formats = {} path_params = {} if 'group' in local_var_params: path_params['group'] = local_var_params['group'] # noqa: E501 if 'version' in local_var_params: path_params['version'] = local_var_params['version'] # noqa: E501 if 'namespace' in local_var_params: path_params['namespace'] = local_var_params['namespace'] # noqa: E501 if 'plural' in local_var_params: path_params['plural'] = local_var_params['plural'] # noqa: E501 if 'name' in local_var_params: path_params['name'] = local_var_params['name'] # noqa: E501 query_params = [] header_params = {} form_params = [] local_var_files = {} body_params = None # HTTP header `Accept` header_params['Accept'] = self.api_client.select_header_accept( ['application/json']) # noqa: E501 # Authentication setting auth_settings = ['BearerToken'] # noqa: E501 > return self.api_client.call_api( '/apis/{group}/{version}/namespaces/{namespace}/{plural}/{name}', 'GET', path_params, query_params, header_params, body=body_params, post_params=form_params, files=local_var_files, response_type='object', # noqa: E501 auth_settings=auth_settings, async_req=local_var_params.get('async_req'), _return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501 _preload_content=local_var_params.get('_preload_content', True), _request_timeout=local_var_params.get('_request_timeout'), collection_formats=collection_formats) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:1739: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202133dd0> resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}/{name}' method = 'GET' path_params = {'group': 'serving.kserve.io', 'name': 'router-custom-route-timeout-pd-8b408578', 'namespace': 'e2e-test-llm-inference-service-d73d44f4', 'plural': 'llminferenceserviceconfigs', ...} query_params = [] header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. :param resource_path: Path to method endpoint. :param method: Method to call. :param path_params: Path parameters in the url. :param query_params: Query parameters in the url. :param header_params: Header parameters to be placed in the request header. :param body: Request body. :param post_params dict: Request post form parameters, for `application/x-www-form-urlencoded`, `multipart/form-data`. :param auth_settings list: Auth Settings names for the request. :param response: Response data type. :param files dict: key -> filename, value -> filepath, for `multipart/form-data`. :param async_req bool: execute request asynchronously :param _return_http_data_only: response data without head status code and headers :param collection_formats: dict of collection formats for path, query, header, and post parameters. :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: If async_req parameter is True, the request will be called asynchronously. The method will return the request thread. If parameter async_req is False or missing, then the method will return the response directly. """ if not async_req: > return self.__call_api(resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202133dd0> resource_path = '/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs/router-custom-route-timeout-pd-8b408578' method = 'GET' path_params = [('group', 'serving.kserve.io'), ('version', 'v1alpha1'), ('namespace', 'e2e-test-llm-inference-service-d73d44f4'), ('plural', 'llminferenceserviceconfigs'), ('name', 'router-custom-route-timeout-pd-8b408578')] query_params = [] header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def __call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict(self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param) ) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = post_params if post_params else [] post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) post_params.extend(self.files_parameters(files)) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url if _host is None: url = self.configuration.host + resource_path else: # use server/host defined in path or operation instead url = _host + resource_path # perform request and return response > response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202133dd0> method = 'GET' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1a.../namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs/router-custom-route-timeout-pd-8b408578' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} post_params = [], body = None, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): """Makes the HTTP request using RESTClient.""" if method == "GET": > return self.rest_client.GET(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:373: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202133e50> url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1a.../namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs/router-custom-route-timeout-pd-8b408578' headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} query_params = [], _preload_content = True, _request_timeout = None def GET(self, url, headers=None, query_params=None, _preload_content=True, _request_timeout=None): > return self.request("GET", url, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout, query_params=query_params) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:244: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202133e50> method = 'GET' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1a.../namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs/router-custom-route-timeout-pd-8b408578' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = None, post_params = {}, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Perform requests. :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] if post_params and body: raise ApiValueError( "body parameter cannot be used with post_params parameter." ) post_params = post_params or {} headers = headers or {} timeout = None if _request_timeout: if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821 timeout = urllib3.Timeout(total=_request_timeout) elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): timeout = urllib3.Timeout( connect=_request_timeout[0], read=_request_timeout[1]) if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) if (re.search('json', headers['Content-Type'], re.IGNORECASE) or headers['Content-Type'] == 'application/apply-patch+yaml'): if headers['Content-Type'] == 'application/json-patch+json': if not isinstance(body, list): headers['Content-Type'] = \ 'application/strategic-merge-patch+json' request_body = None if body is not None: request_body = json.dumps(body) r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=False, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by urllib3 will be # overwritten. del headers['Content-Type'] r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=True, preload_content=_preload_content, timeout=timeout, headers=headers) # Pass a `string` parameter directly in the body to support # other content types than Json when `body` argument is # provided in serialized form elif isinstance(body, str) or isinstance(body, bytes): request_body = body r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) # For `GET`, `HEAD` else: r = self.pool_manager.request(method, url, fields=query_params, preload_content=_preload_content, timeout=timeout, headers=headers) except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) # In the python 3, the response.data is bytes. # we need to decode it to string. if six.PY3: r.data = r.data.decode('utf8') # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: > raise ApiException(http_resp=r) E kubernetes.client.exceptions.ApiException: (404) E Reason: Not Found E HTTP response headers: HTTPHeaderDict({'Audit-Id': '3e266d16-7bda-4ce7-b797-479ff8586efd', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': 'ff0b8d77-f0bc-468d-a86a-0524cad8e456', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'a7ead034-f650-4220-8a0e-0379eadce096', 'Date': 'Tue, 28 Jul 2026 17:24:07 GMT', 'Content-Length': '336'}) E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"llminferenceserviceconfigs.serving.kserve.io \"router-custom-route-timeout-pd-8b408578\" not found","reason":"NotFound","details":{"name":"router-custom-route-timeout-pd-8b408578","group":"serving.kserve.io","kind":"llminferenceserviceconfigs"},"code":404} ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException During handling of the above exception, another exception occurred: request = <SubRequest 'ensure_gateway_proxy_memory' for <Function test_llm_inference_service[router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]>> @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(request): """After test setup creates gateways, patch them for proxy memory.""" if not GATEWAY_PROXY_MEMORY: return # Let test_case (llmisvc) create gateways first if "test_case" in request.fixturenames: > request.getfixturevalue("test_case") common/gateway_proxy_istio.py:183: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ request = <SubRequest 'test_case' for <Function test_llm_inference_service[router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]>> test_namespace = 'e2e-test-llm-inference-service-d73d44f4' @pytest.fixture(scope="function") def test_case(request, test_namespace): tc = request.param ns = test_namespace inject_k8s_proxy() kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config"), client_configuration=client.Configuration(), ) tc.namespace = ns for peer in tc.peers: peer.namespace = ns for func in tc.before_test: func(tc) > _setup_test_case_service(kserve_client, tc, request.node.name, namespace=ns) llmisvc/fixtures.py:1611: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc201e70610> tc = TestCase(base_refs=['router-custom-route-timeout-pd', 'scheduler-managed', 'workload-pd-cpu', 'model-fb-opt-125m'], pr...inference-service-d73d44f4', before_test=[], after_test=[], peers=[], llm_service=None, model_name='facebook/opt-125m') test_node_name = 'test_llm_inference_service[router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]' namespace = 'e2e-test-llm-inference-service-d73d44f4', peer_index = None def _setup_test_case_service( kserve_client, tc, test_node_name, namespace, peer_index=None ): """Create LLMInferenceServiceConfigs and build the LLMInferenceService for a TestCase. Returns a list of created config names for cleanup tracking. """ missing_refs = [ ref for ref in tc.base_refs if ref not in LLMINFERENCESERVICE_CONFIGS ] if missing_refs: raise ValueError( f"Missing base_refs in LLMINFERENCESERVICE_CONFIGS: {missing_refs}" ) if not tc.service_name: suffix = f"-peer-{peer_index}" if peer_index is not None else "" tc.service_name = generate_service_name(test_node_name + suffix, tc.base_refs) if tc.model_name == "default/model": tc.model_name = _get_model_name_from_configs(tc.base_refs) elif "{namespace}" in tc.model_name: tc.model_name = tc.model_name.format(namespace=namespace) created_configs = [] unique_base_refs = [] for base_ref in tc.base_refs: unique_config_name = generate_k8s_safe_suffix(base_ref, [tc.service_name]) unique_base_refs.append(unique_config_name) config = LLMINFERENCESERVICE_CONFIGS[base_ref] spec = config(namespace) if callable(config) else copy.deepcopy(config) unique_config_body = { "apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceServiceConfig", "metadata": { "name": unique_config_name, "namespace": namespace, }, "spec": spec, } > _create_or_update_llmisvc_config(kserve_client, unique_config_body, namespace) llmisvc/fixtures.py:1573: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fc201e70610> llm_config = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} namespace = 'e2e-test-llm-inference-service-d73d44f4' def _create_or_update_llmisvc_config(kserve_client, llm_config, namespace=None): """Create or update an LLMInferenceServiceConfig resource.""" version = llm_config["apiVersion"].split("/")[1] if namespace is None: namespace = llm_config.get("metadata", {}).get("namespace", "default") name = llm_config.get("metadata", {}).get("name") if not name: raise ValueError("LLMInferenceServiceConfig must have a name in metadata") logger.info(f"Checking LLMInferenceServiceConfig {name} in namespace {namespace}") try: existing_config = kserve_client.api_instance.get_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, name, ) llm_config["metadata"] = existing_config["metadata"] outputs = kserve_client.api_instance.replace_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, name, llm_config, ) logger.info(f"✓ Successfully updated LLMInferenceServiceConfig {name}") return outputs except client.rest.ApiException as e: if e.status == 404: # Not found - create it logger.info( f"Resource not found, creating LLMInferenceServiceConfig {name}" ) > outputs = kserve_client.api_instance.create_namespaced_custom_object( constants.KSERVE_GROUP, version, namespace, KSERVE_PLURAL_LLMINFERENCESERVICECONFIG, llm_config, ) llmisvc/fixtures.py:1747: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202133e10> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-d73d44f4' plural = 'llminferenceserviceconfigs' body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} kwargs = {'_return_http_data_only': True} def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501 """create_namespaced_custom_object # noqa: E501 Creates a namespace scoped Custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: The custom resource's group name (required) :param str version: The custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param object body: The JSON schema of the Resource to create. (required) :param str pretty: If 'true', then the output is pretty printed. :param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed :param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. :param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional) :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: object If the method is called asynchronously, returns the request thread. """ kwargs['_return_http_data_only'] = True > return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501 ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api.custom_objects_api.CustomObjectsApi object at 0x7fc202133e10> group = 'serving.kserve.io', version = 'v1alpha1' namespace = 'e2e-test-llm-inference-service-d73d44f4' plural = 'llminferenceserviceconfigs' body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} kwargs = {'_return_http_data_only': True} local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au...e-test-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {...}}}}}}, ...} all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...] key = '_return_http_data_only', val = True, collection_formats = {} path_params = {'group': 'serving.kserve.io', 'namespace': 'e2e-test-llm-inference-service-d73d44f4', 'plural': 'llminferenceserviceconfigs', 'version': 'v1alpha1'} query_params = [] def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501 """create_namespaced_custom_object # noqa: E501 Creates a namespace scoped Custom object # noqa: E501 This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True >>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True) >>> result = thread.get() :param async_req bool: execute request asynchronously :param str group: The custom resource's group name (required) :param str version: The custom resource's version (required) :param str namespace: The custom resource's namespace (required) :param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required) :param object body: The JSON schema of the Resource to create. (required) :param str pretty: If 'true', then the output is pretty printed. :param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed :param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. :param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional) :param _return_http_data_only: response data without head status code and headers :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: tuple(object, status_code(int), headers(HTTPHeaderDict)) If the method is called asynchronously, returns the request thread. """ local_var_params = locals() all_params = [ 'group', 'version', 'namespace', 'plural', 'body', 'pretty', 'dry_run', 'field_manager', 'field_validation' ] all_params.extend( [ 'async_req', '_return_http_data_only', '_preload_content', '_request_timeout' ] ) for key, val in six.iteritems(local_var_params['kwargs']): if key not in all_params: raise ApiTypeError( "Got an unexpected keyword argument '%s'" " to method create_namespaced_custom_object" % key ) local_var_params[key] = val del local_var_params['kwargs'] # verify the required parameter 'group' is set if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501 local_var_params['group'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'version' is set if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501 local_var_params['version'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'namespace' is set if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501 local_var_params['namespace'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'plural' is set if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501 local_var_params['plural'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501 # verify the required parameter 'body' is set if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501 local_var_params['body'] is None): # noqa: E501 raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501 collection_formats = {} path_params = {} if 'group' in local_var_params: path_params['group'] = local_var_params['group'] # noqa: E501 if 'version' in local_var_params: path_params['version'] = local_var_params['version'] # noqa: E501 if 'namespace' in local_var_params: path_params['namespace'] = local_var_params['namespace'] # noqa: E501 if 'plural' in local_var_params: path_params['plural'] = local_var_params['plural'] # noqa: E501 query_params = [] if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501 query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501 if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501 query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501 if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501 query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501 if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501 query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501 header_params = {} form_params = [] local_var_files = {} body_params = None if 'body' in local_var_params: body_params = local_var_params['body'] # HTTP header `Accept` header_params['Accept'] = self.api_client.select_header_accept( ['application/json']) # noqa: E501 # Authentication setting auth_settings = ['BearerToken'] # noqa: E501 > return self.api_client.call_api( '/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST', path_params, query_params, header_params, body=body_params, post_params=form_params, files=local_var_files, response_type='object', # noqa: E501 auth_settings=auth_settings, async_req=local_var_params.get('async_req'), _return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501 _preload_content=local_var_params.get('_preload_content', True), _request_timeout=local_var_params.get('_request_timeout'), collection_formats=collection_formats) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202133dd0> resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}' method = 'POST' path_params = {'group': 'serving.kserve.io', 'namespace': 'e2e-test-llm-inference-service-d73d44f4', 'plural': 'llminferenceserviceconfigs', 'version': 'v1alpha1'} query_params = [] header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def call_api(self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, async_req=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): """Makes the HTTP request (synchronous) and returns deserialized data. To make an async_req request, set the async_req parameter. :param resource_path: Path to method endpoint. :param method: Method to call. :param path_params: Path parameters in the url. :param query_params: Query parameters in the url. :param header_params: Header parameters to be placed in the request header. :param body: Request body. :param post_params dict: Request post form parameters, for `application/x-www-form-urlencoded`, `multipart/form-data`. :param auth_settings list: Auth Settings names for the request. :param response: Response data type. :param files dict: key -> filename, value -> filepath, for `multipart/form-data`. :param async_req bool: execute request asynchronously :param _return_http_data_only: response data without head status code and headers :param collection_formats: dict of collection formats for path, query, header, and post parameters. :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. :return: If async_req parameter is True, the request will be called asynchronously. The method will return the request thread. If parameter async_req is False or missing, then the method will return the response directly. """ if not async_req: > return self.__call_api(resource_path, method, path_params, query_params, header_params, body, post_params, files, response_type, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202133dd0> resource_path = '/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs' method = 'POST' path_params = [('group', 'serving.kserve.io'), ('version', 'v1alpha1'), ('namespace', 'e2e-test-llm-inference-service-d73d44f4'), ('plural', 'llminferenceserviceconfigs')] query_params = [] header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} post_params = [], files = {}, response_type = 'object' auth_settings = ['BearerToken'], _return_http_data_only = True collection_formats = {}, _preload_content = True, _request_timeout = None _host = None def __call_api( self, resource_path, method, path_params=None, query_params=None, header_params=None, body=None, post_params=None, files=None, response_type=None, auth_settings=None, _return_http_data_only=None, collection_formats=None, _preload_content=True, _request_timeout=None, _host=None): config = self.configuration # header parameters header_params = header_params or {} header_params.update(self.default_headers) if self.cookie: header_params['Cookie'] = self.cookie if header_params: header_params = self.sanitize_for_serialization(header_params) header_params = dict(self.parameters_to_tuples(header_params, collection_formats)) # path parameters if path_params: path_params = self.sanitize_for_serialization(path_params) path_params = self.parameters_to_tuples(path_params, collection_formats) for k, v in path_params: # specified safe chars, encode everything resource_path = resource_path.replace( '{%s}' % k, quote(str(v), safe=config.safe_chars_for_path_param) ) # query parameters if query_params: query_params = self.sanitize_for_serialization(query_params) query_params = self.parameters_to_tuples(query_params, collection_formats) # post parameters if post_params or files: post_params = post_params if post_params else [] post_params = self.sanitize_for_serialization(post_params) post_params = self.parameters_to_tuples(post_params, collection_formats) post_params.extend(self.files_parameters(files)) # auth setting self.update_params_for_auth(header_params, query_params, auth_settings) # body if body: body = self.sanitize_for_serialization(body) # request url if _host is None: url = self.configuration.host + resource_path else: # use server/host defined in path or operation instead url = _host + resource_path # perform request and return response > response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.api_client.ApiClient object at 0x7fc202133dd0> method = 'POST' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} post_params = [] body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): """Makes the HTTP request using RESTClient.""" if method == "GET": return self.rest_client.GET(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, headers=headers) elif method == "HEAD": return self.rest_client.HEAD(url, query_params=query_params, _preload_content=_preload_content, _request_timeout=_request_timeout, headers=headers) elif method == "OPTIONS": return self.rest_client.OPTIONS(url, query_params=query_params, headers=headers, _preload_content=_preload_content, _request_timeout=_request_timeout) elif method == "POST": > return self.rest_client.POST(url, query_params=query_params, headers=headers, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202133e50> url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs' headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} query_params = [], post_params = [] body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} _preload_content = True, _request_timeout = None def POST(self, url, headers=None, query_params=None, post_params=None, body=None, _preload_content=True, _request_timeout=None): > return self.request("POST", url, headers=headers, query_params=query_params, post_params=post_params, _preload_content=_preload_content, _request_timeout=_request_timeout, body=body) ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <kubernetes.client.rest.RESTClientObject object at 0x7fc202133e50> method = 'POST' url = 'https://abd8b686d1ff342e4952c8ff19c59c6b-970430a80edceb74.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1alpha1/namespaces/e2e-test-llm-inference-service-d73d44f4/llminferenceserviceconfigs' query_params = [] headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'} body = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceServiceConfig', 'metadata': {'name': 'router-custom-...t-llm-inference-service-d73d44f4'}, 'spec': {'router': {'gateway': {}, 'route': {'http': {'spec': {'rules': [...]}}}}}} post_params = {}, _preload_content = True, _request_timeout = None def request(self, method, url, query_params=None, headers=None, body=None, post_params=None, _preload_content=True, _request_timeout=None): """Perform requests. :param method: http request method :param url: http request url :param query_params: query parameters in the url :param headers: http request headers :param body: request json body, for `application/json` :param post_params: request post parameters, `application/x-www-form-urlencoded` and `multipart/form-data` :param _preload_content: if False, the urllib3.HTTPResponse object will be returned without reading/decoding response data. Default is True. :param _request_timeout: timeout setting for this request. If one number provided, it will be total request timeout. It can also be a pair (tuple) of (connection, read) timeouts. """ method = method.upper() assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT', 'PATCH', 'OPTIONS'] if post_params and body: raise ApiValueError( "body parameter cannot be used with post_params parameter." ) post_params = post_params or {} headers = headers or {} timeout = None if _request_timeout: if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821 timeout = urllib3.Timeout(total=_request_timeout) elif (isinstance(_request_timeout, tuple) and len(_request_timeout) == 2): timeout = urllib3.Timeout( connect=_request_timeout[0], read=_request_timeout[1]) if 'Content-Type' not in headers: headers['Content-Type'] = 'application/json' try: # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']: if query_params: url += '?' + urlencode(query_params) if (re.search('json', headers['Content-Type'], re.IGNORECASE) or headers['Content-Type'] == 'application/apply-patch+yaml'): if headers['Content-Type'] == 'application/json-patch+json': if not isinstance(body, list): headers['Content-Type'] = \ 'application/strategic-merge-patch+json' request_body = None if body is not None: request_body = json.dumps(body) r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501 r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=False, preload_content=_preload_content, timeout=timeout, headers=headers) elif headers['Content-Type'] == 'multipart/form-data': # must del headers['Content-Type'], or the correct # Content-Type which generated by urllib3 will be # overwritten. del headers['Content-Type'] r = self.pool_manager.request( method, url, fields=post_params, encode_multipart=True, preload_content=_preload_content, timeout=timeout, headers=headers) # Pass a `string` parameter directly in the body to support # other content types than Json when `body` argument is # provided in serialized form elif isinstance(body, str) or isinstance(body, bytes): request_body = body r = self.pool_manager.request( method, url, body=request_body, preload_content=_preload_content, timeout=timeout, headers=headers) else: # Cannot generate the request from given parameters msg = """Cannot prepare a request message for provided arguments. Please check that your arguments match declared content type.""" raise ApiException(status=0, reason=msg) # For `GET`, `HEAD` else: r = self.pool_manager.request(method, url, fields=query_params, preload_content=_preload_content, timeout=timeout, headers=headers) except urllib3.exceptions.SSLError as e: msg = "{0}\n{1}".format(type(e).__name__, str(e)) raise ApiException(status=0, reason=msg) if _preload_content: r = RESTResponse(r) # In the python 3, the response.data is bytes. # we need to decode it to string. if six.PY3: r.data = r.data.decode('utf8') # log response body logger.debug("response body: %s", r.data) if not 200 <= r.status <= 299: > raise ApiException(http_resp=r) E kubernetes.client.exceptions.ApiException: (500) E Reason: Internal Server Error E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'ec254936-5f14-42fa-86e9-bae4267dd9cf', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': 'ff0b8d77-f0bc-468d-a86a-0524cad8e456', 'X-Kubernetes-Pf-Prioritylevel-Uid': 'a7ead034-f650-4220-8a0e-0379eadce096', 'Date': 'Tue, 28 Jul 2026 17:24:07 GMT', 'Content-Length': '701'}) E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.v1alpha1.validator\": failed to call webhook: Post \"https://llmisvc-webhook-server-service.kserve.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": EOF","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"llminferenceserviceconfig.kserve-webhook-server.v1alpha1.validator\": failed to call webhook: Post \"https://llmisvc-webhook-server-service.kserve.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceserviceconfig?timeout=10s\": EOF"}]},"code":500} ../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException