{"created": 1784546748.1897497, "duration": 4490.2880964279175, "exitcode": 2, "root": "/workspace/source/test/e2e", "environment": {}, "summary": {"passed": 46, "failed": 5, "total": 51, "collected": 60}, "collectors": [{"nodeid": "explainer/test_art_explainer.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')"}, {"nodeid": "predictor/test_grpc.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')"}, {"nodeid": "predictor/test_torchserve.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')"}], "tests": [{"nodeid": "llmisvc/test_flow_control.py::test_flow_control_smoke[cluster_cpu-cluster_single_node-flow-control-utilization-detector]", "lineno": 46, "outcome": "passed", "keywords": ["test_flow_control_smoke[flow-control-utilization-detector]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "flow_control", "pytestmark", "flow-control-utilization-detector", "llminferenceservice", "llmisvc_core", "test_flow_control.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.6307657590004965, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 80.92240341599972, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03391973700126982, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-custom-template-workload-llmd-simulator]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-custom-template-workload-llmd-simulator]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-custom-template-workload-llmd-simulator", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.7250217339969822, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 74.65466017600193, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04414225399887073, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.5122668889998749, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 129.49631779300034, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04022069500206271, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_flow_control.py::test_flow_control_smoke[cluster_cpu-cluster_single_node-flow-control-concurrency-detector]", "lineno": 46, "outcome": "passed", "keywords": ["test_flow_control_smoke[flow-control-concurrency-detector]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "flow_control", "pytestmark", "flow-control-concurrency-detector", "llminferenceservice", "llmisvc_core", "test_flow_control.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.6552913280029316, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 64.9927446670008, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04071088300042902, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_gateway_section_name.py::test_gateway_section_name_propagation[cluster_single_node-cluster_cpu-with-section-name]", "lineno": 131, "outcome": "passed", "keywords": ["test_gateway_section_name_propagation[with-section-name]", "parametrize", "llmd_simulator", "cluster_single_node", "cluster_cpu", "pytestmark", "with-section-name", "llminferenceservice", "llmisvc_core", "test_gateway_section_name.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.19630439300090075, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 9.227568899998005, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.038716353999916464, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_gateway_section_name.py::test_gateway_section_name_propagation[cluster_single_node-cluster_cpu-without-section-name]", "lineno": 131, "outcome": "passed", "keywords": ["test_gateway_section_name_propagation[without-section-name]", "parametrize", "llmd_simulator", "cluster_single_node", "cluster_cpu", "pytestmark", "without-section-name", "llminferenceservice", "llmisvc_core", "test_gateway_section_name.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.19537503500032471, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 20.771764345998236, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04654356100218138, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_auth.py::test_llm_auth_enabled_requires_token[cluster_cpu-cluster_single_node-auth-enabled-default]", "lineno": 221, "outcome": "failed", "keywords": ["test_llm_auth_enabled_requires_token[auth-enabled-default]", "parametrize", "auth", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "auth-enabled-default", "llmisvc_core", "test_llm_auth.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.32514106199960224, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 279.45792033399994, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_auth.py", "lineno": 336, "message": "AssertionError: Expected 200 with token, got 403: \nassert 403 == 200\n + where 403 = .status_code"}, "traceback": [{"path": "llmisvc/test_llm_auth.py", "lineno": 336, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m'], prompt='KServe is a', service_name=... {'name': 'model-fb-opt-125m-auth-enabled-89f54b63'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n @pytest.mark.llminferenceservice\n @pytest.mark.auth\n @pytest.mark.parametrize(\n \"test_case\",\n [\n pytest.param(\n TestCase(\n base_refs=[\n \"router-managed\",\n \"workload-single-cpu\",\n \"model-fb-opt-125m\",\n ],\n prompt=\"KServe is a\",\n service_name=\"auth-enabled-test\",\n ),\n marks=[\n pytest.mark.cluster_cpu,\n pytest.mark.cluster_single_node,\n ],\n id=\"auth-enabled-default\",\n ),\n ],\n indirect=[\"test_case\"],\n ids=generate_test_id,\n )\n @log_execution\n def test_llm_auth_enabled_requires_token(test_case: TestCase): # noqa: F811\n \"\"\"\n Test that when auth is enabled (default):\n - Requests WITH valid token succeed\n - Requests WITHOUT token are rejected (401/403)\n \"\"\"\n inject_k8s_proxy()\n \n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n client_configuration=client.Configuration(),\n )\n \n service_name = test_case.llm_service.metadata.name\n sa_name = f\"{service_name}-test-sa\"\n test_failed = False\n \n # Enable auth for this test\n if not test_case.llm_service.metadata.annotations:\n test_case.llm_service.metadata.annotations = {}\n test_case.llm_service.metadata.annotations[\n \"security.opendatahub.io/enable-auth\"\n ] = \"true\"\n \n try:\n # Create LLMInferenceService\n create_llmisvc(kserve_client, test_case.llm_service)\n wait_for_llm_isvc_ready(\n kserve_client, test_case.llm_service, test_case.wait_timeout\n )\n \n # Create ServiceAccount with get+post access (required for inference-access and endpoint-access)\n token = create_service_account_with_inference_access(\n kserve_client, sa_name, service_name\n )\n \n service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n completion_url = f\"{service_url}/v1/completions\"\n test_payload = {\n \"model\": test_case.model_name,\n \"prompt\": test_case.prompt,\n \"max_tokens\": test_case.max_tokens,\n }\n \n # Test 1: Request WITHOUT token should fail\n logger.info(\"Testing request WITHOUT token (should fail)\")\n response_no_token = requests.post(\n completion_url,\n headers={\"Content-Type\": \"application/json\"},\n json=test_payload,\n timeout=30,\n )\n assert response_no_token.status_code in [\n 401,\n 403,\n ], (\n f\"Expected 401/403 without token, got {response_no_token.status_code}: {response_no_token.text}\"\n )\n logger.info(\n f\"\u2705 Request without token rejected: {response_no_token.status_code}\"\n )\n \n # Test 2: Request WITH valid token should succeed.\n # Retry to handle propagation delay \u2014 RBAC changes and Authorino auth policy\n # caches may not reflect new bindings immediately after creation.\n logger.info(\"Testing request WITH valid token (should succeed)\")\n token_headers = {\n \"Content-Type\": \"application/json\",\n \"Authorization\": f\"Bearer {token}\",\n }\n response_with_token = None\n for attempt in range(24): # up to ~120s\n response_with_token = requests.post(\n completion_url,\n headers=token_headers,\n json=test_payload,\n timeout=test_case.response_timeout,\n )\n if response_with_token.status_code == 200:\n break\n if response_with_token.status_code in [401, 403]:\n logger.info(\n f\"Attempt {attempt + 1}: got {response_with_token.status_code}, \"\n \"waiting for RBAC propagation...\"\n )\n time.sleep(5)\n else:\n break\n> assert response_with_token.status_code == 200, (\n f\"Expected 200 with token, got {response_with_token.status_code}: {response_with_token.text}\"\n )\nE AssertionError: Expected 200 with token, got 403: \nE assert 403 == 200\nE + where 403 = .status_code\n\nllmisvc/test_llm_auth.py:336: AssertionError"}, "teardown": {"duration": 0.03586689499934437, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.38492557300196495, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 64.557600250002, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04062265200263937, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.31876145700152847, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 120.45685683099873, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04351159900033963, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator0]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator0]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator0", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2879817950015422, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 63.17007325400118, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.039315873000305146, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator1]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator1]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "model_routing", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator1", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.3939361859993369, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 134.5805157520008, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03590195099968696, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_auth.py::test_llm_auth_invalid_token_rejected[cluster_cpu-cluster_single_node-auth-invalid-token]", "lineno": 385, "outcome": "passed", "keywords": ["test_llm_auth_invalid_token_rejected[auth-invalid-token]", "parametrize", "auth", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "auth-invalid-token", "llmisvc_core", "test_llm_auth.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.5986054300010437, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 174.3155434470027, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03355868200014811, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator2]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator2]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "model_routing", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator2", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.38600546599991503, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 132.96084153500124, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04092206900168094, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_auth.py::test_llm_auth_disabled_no_token_required[cluster_cpu-cluster_single_node-auth-disabled]", "lineno": 521, "outcome": "passed", "keywords": ["test_llm_auth_disabled_no_token_required[auth-disabled]", "parametrize", "auth", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "auth-disabled", "llmisvc_core", "test_llm_auth.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 1.0425890910009912, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 161.1529734600008, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04999860099997022, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "model_routing", "lora", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.3099744399987685, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 144.90581550800198, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03650800599643844, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_autoscaling_wva.py::test_llm_autoscaling_hpa_deployment[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator-no-replicas-prometheus-scrape-scaling-hpa]", "lineno": 505, "outcome": "failed", "keywords": ["test_llm_autoscaling_hpa_deployment[router-managed-workload-llmd-simulator-no-replicas-prometheus-scrape-scaling-hpa]", "parametrize", "autoscaling_hpa", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-no-replicas-prometheus-scrape-scaling-hpa", "llminferenceservice", "llmisvc_autoscaling", "autoscaling_wva", "test_llm_autoscaling_wva.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.3254837360000238, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 905.9541693109968, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]"}, "traceback": [{"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 540, "message": ""}, {"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 480, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1255, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1266, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n @pytest.mark.autoscaling_hpa\n @pytest.mark.parametrize(\n \"test_case\",\n [\n pytest.param(\n TestCase(\n base_refs=[\n \"router-managed\",\n \"workload-llmd-simulator-no-replicas\",\n \"prometheus-scrape\",\n \"scaling-hpa\",\n ],\n prompt=\"KServe is a\",\n service_name=\"autoscale-hpa-deploy\",\n ),\n marks=[\n pytest.mark.cluster_cpu,\n pytest.mark.cluster_single_node,\n pytest.mark.llmd_simulator,\n ],\n ),\n ],\n indirect=[\"test_case\"],\n ids=generate_test_id,\n )\n @log_execution\n def test_llm_autoscaling_hpa_deployment(test_case: TestCase):\n \"\"\"HPA + Deployment: VA and HPA exist; pods scale up under load.\"\"\"\n inject_k8s_proxy()\n kserve_client = _new_kserve_client()\n service_name = test_case.llm_service.metadata.name\n ns = test_case.namespace\n \n try:\n> _create_and_wait(kserve_client, test_case)\n\nllmisvc/test_llm_autoscaling_wva.py:540: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-hpa'], prom... {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n def _create_and_wait(kserve_client, test_case):\n \"\"\"Create LLMISVC and wait for it to be ready.\"\"\"\n create_llmisvc(kserve_client, test_case.llm_service)\n> wait_for_llm_isvc_ready(\n kserve_client, test_case.llm_service, test_case.wait_timeout\n )\n\nllmisvc/test_llm_autoscaling_wva.py:480: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (, {'api_version': 'serving.kserve.io/v1alpha1',\n 'kin...e-hpa-4c186bcf'},\n {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]},\n 'status': None}, 900)\nkwargs = {}, func_name = 'wait_for_llm_isvc_ready'\ntimestamp_start = '2026-07-20T10:24:15.869788', start_time = 1784543055.87021\nduration = 900.8121633529663, timestamp_end = '2026-07-20T10:39:16.682376'\n\n @functools.wraps(func)\n def wrapper(*args, **kwargs):\n func_name = func.__name__\n \n timestamp_start = datetime.now().isoformat()\n logger.info(\n f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n )\n start_time = time.time()\n \n try:\n> result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ngiven = {'api_version': 'serving.kserve.io/v1alpha1',\n 'kind': 'LLMInferenceService',\n 'metadata': {'annotations': None,\n ...toscale-hpa-4c186bcf'},\n {'name': 'scaling-hpa-autoscale-hpa-deplo-347a3180'}]},\n 'status': None}\ntimeout_seconds = 900\n\n @log_execution\n def wait_for_llm_isvc_ready(\n kserve_client: KServeClient,\n given: V1alpha1LLMInferenceService,\n timeout_seconds: int = 900,\n ) -> str:\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\n return True\n \n> return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0)\n\nllmisvc/test_llm_inference_service.py:1255: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = .assert_llm_isvc_ready at 0x7f8d21052700>\ntimeout = 900, interval = 1.0\n\n def wait_for(\n assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n ) -> Any:\n \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n deadline = time.time() + timeout\n last_msg = None\n while True:\n try:\n> return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1266: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n> raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\nE AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:24:21Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-deployment-9ba6f3f4/autoscale-hpa-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]\n\nllmisvc/test_llm_inference_service.py:1250: AssertionError"}, "teardown": {"duration": 0.03493561099821818, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "model_routing", "lora", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.30115115599983255, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 119.33634751000136, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04223833599826321, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-pvc]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-pvc]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "pvc_storage", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-pvc", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 20.52725234600075, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 164.7101233479989, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0382940689996758, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-pd-cpu-model-pvc]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-pd-cpu-model-pvc]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "pvc_storage", "__wrapped__", "pytestmark", "router-managed-workload-pd-cpu-model-pvc", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 20.548104110999702, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 205.1459928430013, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.041076432997215306, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_multi_node-router-managed-workload-simulated-dp-ep-cpu-model-pvc]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-simulated-dp-ep-cpu-model-pvc]", "parametrize", "asyncio", "cluster_cpu", "cluster_multi_node", "pvc_storage", "__wrapped__", "pytestmark", "router-managed-workload-simulated-dp-ep-cpu-model-pvc", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 20.51230639500136, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 317.3254756210008, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04386899800010724, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_autoscaling_wva.py::test_llm_autoscaling_keda_deployment[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator-no-replicas-prometheus-scrape-scaling-keda]", "lineno": 569, "outcome": "failed", "keywords": ["test_llm_autoscaling_keda_deployment[router-managed-workload-llmd-simulator-no-replicas-prometheus-scrape-scaling-keda]", "parametrize", "autoscaling_keda", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-no-replicas-prometheus-scrape-scaling-keda", "llminferenceservice", "llmisvc_autoscaling", "autoscaling_wva", "test_llm_autoscaling_wva.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.781799158998183, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 905.4182359279985, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]"}, "traceback": [{"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 604, "message": ""}, {"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 480, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1255, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1266, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n @pytest.mark.autoscaling_keda\n @pytest.mark.parametrize(\n \"test_case\",\n [\n pytest.param(\n TestCase(\n base_refs=[\n \"router-managed\",\n \"workload-llmd-simulator-no-replicas\",\n \"prometheus-scrape\",\n \"scaling-keda\",\n ],\n prompt=\"KServe is a\",\n service_name=\"autoscale-keda-deploy\",\n ),\n marks=[\n pytest.mark.cluster_cpu,\n pytest.mark.cluster_single_node,\n pytest.mark.llmd_simulator,\n ],\n ),\n ],\n indirect=[\"test_case\"],\n ids=generate_test_id,\n )\n @log_execution\n def test_llm_autoscaling_keda_deployment(test_case: TestCase):\n \"\"\"KEDA + Deployment: VA and ScaledObject exist; no HPA; pods scale up under load.\"\"\"\n inject_k8s_proxy()\n kserve_client = _new_kserve_client()\n service_name = test_case.llm_service.metadata.name\n ns = test_case.namespace\n \n try:\n> _create_and_wait(kserve_client, test_case)\n\nllmisvc/test_llm_autoscaling_wva.py:604: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-no-replicas', 'prometheus-scrape', 'scaling-keda'], pro... {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n def _create_and_wait(kserve_client, test_case):\n \"\"\"Create LLMISVC and wait for it to be ready.\"\"\"\n create_llmisvc(kserve_client, test_case.llm_service)\n> wait_for_llm_isvc_ready(\n kserve_client, test_case.llm_service, test_case.wait_timeout\n )\n\nllmisvc/test_llm_autoscaling_wva.py:480: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (, {'api_version': 'serving.kserve.io/v1alpha1',\n 'kin...e-ked-101f2a9d'},\n {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]},\n 'status': None}, 900)\nkwargs = {}, func_name = 'wait_for_llm_isvc_ready'\ntimestamp_start = '2026-07-20T10:39:22.870344', start_time = 1784543962.8706155\nduration = 900.0523600578308, timestamp_end = '2026-07-20T10:54:22.922983'\n\n @functools.wraps(func)\n def wrapper(*args, **kwargs):\n func_name = func.__name__\n \n timestamp_start = datetime.now().isoformat()\n logger.info(\n f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n )\n start_time = time.time()\n \n try:\n> result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ngiven = {'api_version': 'serving.kserve.io/v1alpha1',\n 'kind': 'LLMInferenceService',\n 'metadata': {'annotations': None,\n ...toscale-ked-101f2a9d'},\n {'name': 'scaling-keda-autoscale-keda-dep-1ac84077'}]},\n 'status': None}\ntimeout_seconds = 900\n\n @log_execution\n def wait_for_llm_isvc_ready(\n kserve_client: KServeClient,\n given: V1alpha1LLMInferenceService,\n timeout_seconds: int = 900,\n ) -> str:\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\n return True\n \n> return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0)\n\nllmisvc/test_llm_inference_service.py:1255: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = .assert_llm_isvc_ready at 0x7f8d20616a20>\ntimeout = 900, interval = 1.0\n\n def wait_for(\n assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n ) -> Any:\n \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n deadline = time.time() + timeout\n last_msg = None\n while True:\n try:\n> return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1266: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n> raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\nE AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:39:29Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-deployment-b2150d0b/autoscale-keda-deploy-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]\n\nllmisvc/test_llm_inference_service.py:1250: AssertionError"}, "teardown": {"duration": 0.03542472499975702, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_config_deletion.py::test_config_finalizer_added", "lineno": 191, "outcome": "passed", "keywords": ["test_config_finalizer_added", "cluster_single_node", "cluster_cpu", "__wrapped__", "pytestmark", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_config_deletion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.1685463029971288, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 2.0811255449989403, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03227902100115898, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_config_deletion.py::test_config_deletion_blocked_when_referenced", "lineno": 224, "outcome": "passed", "keywords": ["test_config_deletion_blocked_when_referenced", "cluster_single_node", "cluster_cpu", "__wrapped__", "pytestmark", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_config_deletion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.15701621799962595, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 12.366584210998553, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03530361599769094, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_config_deletion.py::test_config_deletion_allowed_when_unreferenced", "lineno": 314, "outcome": "passed", "keywords": ["test_config_deletion_allowed_when_unreferenced", "cluster_single_node", "cluster_cpu", "__wrapped__", "pytestmark", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_config_deletion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2245806850005465, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 4.149335599999176, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.028198184998473153, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_config_deletion.py::test_config_deletion_unblocked_after_service_deleted", "lineno": 349, "outcome": "passed", "keywords": ["test_config_deletion_unblocked_after_service_deleted", "cluster_single_node", "cluster_cpu", "__wrapped__", "pytestmark", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_config_deletion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.15555380400110153, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 2.2927714369980094, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03301354099676246, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_config_deletion.py::test_well_known_config_deletion_prevented_by_webhook", "lineno": 431, "outcome": "passed", "keywords": ["test_well_known_config_deletion_prevented_by_webhook", "cluster_single_node", "cluster_cpu", "__wrapped__", "pytestmark", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_config_deletion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.030247184997278964, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.06957424099891796, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.00021207399913691916, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_config_deletion.py::test_well_known_config_deletion_blocked_by_implicit_reference", "lineno": 465, "outcome": "passed", "keywords": ["test_well_known_config_deletion_blocked_by_implicit_reference", "cluster_single_node", "cluster_cpu", "__wrapped__", "pytestmark", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_config_deletion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.14682776200061198, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 14.339587455000583, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03561794900087989, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_v1alpha1_to_v1alpha2_conversion", "lineno": 210, "outcome": "passed", "keywords": ["test_v1alpha1_to_v1alpha2_conversion", "cluster_single_node", "cluster_cpu", "pytestmark", "llminferenceservice", "llmisvc_core", "TestLLMInferenceServiceConversion", "conversion", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.17529341999761527, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.09119865500179003, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.4030789940006798, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_v1alpha2_to_v1alpha1_conversion", "lineno": 301, "outcome": "passed", "keywords": ["test_v1alpha2_to_v1alpha1_conversion", "cluster_single_node", "cluster_cpu", "pytestmark", "llminferenceservice", "llmisvc_core", "TestLLMInferenceServiceConversion", "conversion", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.15989603199705016, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.3930862449997221, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.1503634129985585, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_criticality_preservation_via_annotations", "lineno": 392, "outcome": "passed", "keywords": ["test_criticality_preservation_via_annotations", "cluster_single_node", "cluster_cpu", "pytestmark", "llminferenceservice", "llmisvc_core", "TestLLMInferenceServiceConversion", "conversion", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.16736590200162027, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.8811243279997143, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.3459843769996951, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_lora_criticality_preservation", "lineno": 529, "outcome": "passed", "keywords": ["test_lora_criticality_preservation", "cluster_single_node", "cluster_cpu", "pytestmark", "llminferenceservice", "llmisvc_core", "TestLLMInferenceServiceConversion", "conversion", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.21148235000146087, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.6417078430022229, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.33848156800013385, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_round_trip_conversion_preserves_fields", "lineno": 678, "outcome": "passed", "keywords": ["test_round_trip_conversion_preserves_fields", "cluster_single_node", "cluster_cpu", "pytestmark", "llminferenceservice", "llmisvc_core", "TestLLMInferenceServiceConversion", "conversion", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2126897439993627, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.6541521910003212, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.7219162299988966, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_stop.py::test_llm_stop_feature[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 39, "outcome": "passed", "keywords": ["test_llm_stop_feature[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_llm_inference_service_stop.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.6991715650001424, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 302.9395816060023, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04495972500080825, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_lora_adapters.py::test_llm_with_lora_adapters[cluster_cpu-single-lora-adapter-hf]", "lineno": 209, "outcome": "passed", "keywords": ["test_llm_with_lora_adapters[single-lora-adapter-hf]", "parametrize", "cluster_cpu", "lora", "__wrapped__", "pytestmark", "single-lora-adapter-hf", "llminferenceservice", "llmisvc_core", "test_llm_lora_adapters.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.165230086997326, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 147.48277770099958, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.045309034998354036, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_lora_adapters.py::test_llm_with_lora_adapters[cluster_cpu-multiple-lora-adapters]", "lineno": 209, "outcome": "passed", "keywords": ["test_llm_with_lora_adapters[multiple-lora-adapters]", "parametrize", "cluster_cpu", "lora", "__wrapped__", "pytestmark", "multiple-lora-adapters", "llminferenceservice", "llmisvc_core", "test_llm_lora_adapters.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.17741754600137938, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 165.44089056400117, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.042467012000997784, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_tls.py::test_llm_tls_resources[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 91, "outcome": "passed", "keywords": ["test_llm_tls_resources[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "llmisvc_core", "test_llm_tls.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2704348959996423, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 157.21005638100178, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.035415772999840556, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_prestop_hook.py::test_prestop_hook[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 40, "outcome": "passed", "keywords": ["test_prestop_hook[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_prestop_hook.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.34855504400184145, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 208.9963353580024, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.039118584001698764, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_autoscaling_wva.py::test_llm_autoscaling_hpa_lws[cluster_cpu-cluster_multi_node-router-managed-workload-llmd-simulator-lws-prometheus-scrape-scaling-hpa]", "lineno": 633, "outcome": "failed", "keywords": ["test_llm_autoscaling_hpa_lws[router-managed-workload-llmd-simulator-lws-prometheus-scrape-scaling-hpa]", "parametrize", "autoscaling_hpa", "cluster_cpu", "cluster_multi_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-lws-prometheus-scrape-scaling-hpa", "llminferenceservice", "llmisvc_autoscaling", "autoscaling_wva", "test_llm_autoscaling_wva.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2929570349988353, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 905.8621503909999, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:54:44Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]"}, "traceback": [{"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 668, "message": ""}, {"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 480, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1255, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1266, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n @pytest.mark.autoscaling_hpa\n @pytest.mark.parametrize(\n \"test_case\",\n [\n pytest.param(\n TestCase(\n base_refs=[\n \"router-managed\",\n \"workload-llmd-simulator-lws\",\n \"prometheus-scrape\",\n \"scaling-hpa\",\n ],\n prompt=\"KServe is a\",\n service_name=\"autoscale-hpa-lws\",\n ),\n marks=[\n pytest.mark.cluster_cpu,\n pytest.mark.cluster_multi_node,\n pytest.mark.llmd_simulator,\n ],\n ),\n ],\n indirect=[\"test_case\"],\n ids=generate_test_id,\n )\n @log_execution\n def test_llm_autoscaling_hpa_lws(test_case: TestCase):\n \"\"\"HPA + LWS: VA and HPA exist; pods scale under load.\"\"\"\n inject_k8s_proxy()\n kserve_client = _new_kserve_client()\n service_name = test_case.llm_service.metadata.name\n ns = test_case.namespace\n \n try:\n> _create_and_wait(kserve_client, test_case)\n\nllmisvc/test_llm_autoscaling_wva.py:668: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-hpa'], prompt='KSer... {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n def _create_and_wait(kserve_client, test_case):\n \"\"\"Create LLMISVC and wait for it to be ready.\"\"\"\n create_llmisvc(kserve_client, test_case.llm_service)\n> wait_for_llm_isvc_ready(\n kserve_client, test_case.llm_service, test_case.wait_timeout\n )\n\nllmisvc/test_llm_autoscaling_wva.py:480: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (, {'api_version': 'serving.kserve.io/v1alpha1',\n 'kin...ale-hpa-b29acdba'},\n {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]},\n 'status': None}, 900)\nkwargs = {}, func_name = 'wait_for_llm_isvc_ready'\ntimestamp_start = '2026-07-20T10:54:28.467008', start_time = 1784544868.4673026\nduration = 900.705540895462, timestamp_end = '2026-07-20T11:09:29.172846'\n\n @functools.wraps(func)\n def wrapper(*args, **kwargs):\n func_name = func.__name__\n \n timestamp_start = datetime.now().isoformat()\n logger.info(\n f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n )\n start_time = time.time()\n \n try:\n> result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ngiven = {'api_version': 'serving.kserve.io/v1alpha1',\n 'kind': 'LLMInferenceService',\n 'metadata': {'annotations': None,\n ...autoscale-hpa-b29acdba'},\n {'name': 'scaling-hpa-autoscale-hpa-lws-b344a3ff'}]},\n 'status': None}\ntimeout_seconds = 900\n\n @log_execution\n def wait_for_llm_isvc_ready(\n kserve_client: KServeClient,\n given: V1alpha1LLMInferenceService,\n timeout_seconds: int = 900,\n ) -> str:\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\n return True\n \n> return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0)\n\nllmisvc/test_llm_inference_service.py:1255: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = .assert_llm_isvc_ready at 0x7f8d206177e0>\ntimeout = 900, interval = 1.0\n\n def wait_for(\n assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n ) -> Any:\n \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n deadline = time.time() + timeout\n last_msg = None\n while True:\n try:\n> return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1266: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n> raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\nE AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T10:54:44Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-20T10:54:33Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-hpa-lws-d4cbcfd2/autoscale-hpa-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]\n\nllmisvc/test_llm_inference_service.py:1250: AssertionError"}, "teardown": {"duration": 0.035866769998392556, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_rolling_upgrade.py::test_rolling_upgrade_coordination[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator-model-fb-opt-125m]", "lineno": 40, "outcome": "passed", "keywords": ["test_rolling_upgrade_coordination[router-managed-workload-llmd-simulator-model-fb-opt-125m]", "parametrize", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_rolling_upgrade.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2977600150006765, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 73.13538181200056, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04700623900134815, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_storage_version_migration.py::TestStorageVersionMigration::test_storage_version_migration_after_simulated_upgrade", "lineno": 111, "outcome": "passed", "keywords": ["test_storage_version_migration_after_simulated_upgrade", "cluster_single_node", "cluster_cpu", "pytestmark", "llminferenceservice", "llmisvc_core", "TestStorageVersionMigration", "conversion", "test_storage_version_migration.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.3052127049995761, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 64.87822607699854, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 2.3351412779993552, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "custom_gateway", "__wrapped__", "pytestmark", "router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 2.751489941001637, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 295.1879549180012, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04060063299766625, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-pd-cpu-model-fb-opt-125m]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-pd-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-pd-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.3045720779991825, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 203.71946244399805, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04454198399980669, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.41609402500034776, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 212.57820911899762, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04140225900118821, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_autoscaling_wva.py::test_llm_autoscaling_keda_lws[cluster_cpu-cluster_multi_node-router-managed-workload-llmd-simulator-lws-prometheus-scrape-scaling-keda]", "lineno": 691, "outcome": "failed", "keywords": ["test_llm_autoscaling_keda_lws[router-managed-workload-llmd-simulator-lws-prometheus-scrape-scaling-keda]", "parametrize", "autoscaling_keda", "cluster_cpu", "cluster_multi_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-lws-prometheus-scrape-scaling-keda", "llminferenceservice", "llmisvc_autoscaling", "autoscaling_wva", "test_llm_autoscaling_wva.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.5499165119981626, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 905.8172666190003, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T11:09:53Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]"}, "traceback": [{"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 726, "message": ""}, {"path": "llmisvc/test_llm_autoscaling_wva.py", "lineno": 480, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1255, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1266, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1250, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n @pytest.mark.autoscaling_keda\n @pytest.mark.parametrize(\n \"test_case\",\n [\n pytest.param(\n TestCase(\n base_refs=[\n \"router-managed\",\n \"workload-llmd-simulator-lws\",\n \"prometheus-scrape\",\n \"scaling-keda\",\n ],\n prompt=\"KServe is a\",\n service_name=\"autoscale-keda-lws\",\n ),\n marks=[\n pytest.mark.cluster_cpu,\n pytest.mark.cluster_multi_node,\n pytest.mark.llmd_simulator,\n ],\n ),\n ],\n indirect=[\"test_case\"],\n ids=generate_test_id,\n )\n @log_execution\n def test_llm_autoscaling_keda_lws(test_case: TestCase):\n \"\"\"KEDA + LWS: VA and ScaledObject exist; pods scale under load.\"\"\"\n inject_k8s_proxy()\n kserve_client = _new_kserve_client()\n service_name = test_case.llm_service.metadata.name\n ns = test_case.namespace\n \n try:\n> _create_and_wait(kserve_client, test_case)\n\nllmisvc/test_llm_autoscaling_wva.py:726: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ntest_case = TestCase(base_refs=['router-managed', 'workload-llmd-simulator-lws', 'prometheus-scrape', 'scaling-keda'], prompt='KSe... {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n def _create_and_wait(kserve_client, test_case):\n \"\"\"Create LLMISVC and wait for it to be ready.\"\"\"\n create_llmisvc(kserve_client, test_case.llm_service)\n> wait_for_llm_isvc_ready(\n kserve_client, test_case.llm_service, test_case.wait_timeout\n )\n\nllmisvc/test_llm_autoscaling_wva.py:480: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (, {'api_version': 'serving.kserve.io/v1alpha1',\n 'kin...e-ked-231d315d'},\n {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]},\n 'status': None}, 900)\nkwargs = {}, func_name = 'wait_for_llm_isvc_ready'\ntimestamp_start = '2026-07-20T11:09:35.035005', start_time = 1784545775.0353525\nduration = 900.581330537796, timestamp_end = '2026-07-20T11:24:35.616685'\n\n @functools.wraps(func)\n def wrapper(*args, **kwargs):\n func_name = func.__name__\n \n timestamp_start = datetime.now().isoformat()\n logger.info(\n f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n )\n start_time = time.time()\n \n try:\n> result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = \ngiven = {'api_version': 'serving.kserve.io/v1alpha1',\n 'kind': 'LLMInferenceService',\n 'metadata': {'annotations': None,\n ...toscale-ked-231d315d'},\n {'name': 'scaling-keda-autoscale-keda-lws-1337f511'}]},\n 'status': None}\ntimeout_seconds = 900\n\n @log_execution\n def wait_for_llm_isvc_ready(\n kserve_client: KServeClient,\n given: V1alpha1LLMInferenceService,\n timeout_seconds: int = 900,\n ) -> str:\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\n return True\n \n> return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0)\n\nllmisvc/test_llm_inference_service.py:1255: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = .assert_llm_isvc_ready at 0x7f8d20616ca0>\ntimeout = 900, interval = 1.0\n\n def wait_for(\n assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n ) -> Any:\n \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n deadline = time.time() + timeout\n last_msg = None\n while True:\n try:\n> return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1266: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n def assert_llm_isvc_ready():\n out = get_llmisvc(\n kserve_client,\n given.metadata.name,\n given.metadata.namespace,\n given.api_version.split(\"/\")[1],\n )\n \n if \"status\" not in out:\n raise AssertionError(\"No status found in LLM inference service\")\n \n status = out[\"status\"]\n if \"conditions\" not in status:\n raise AssertionError(\"No conditions found in status\")\n \n expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n got_true_conditions = set()\n \n conditions = status[\"conditions\"]\n \n for condition in conditions:\n if condition.get(\"status\") == \"True\":\n got_true_conditions.add(condition.get(\"type\"))\n \n missing_conditions = expected_true_conditions - got_true_conditions\n if missing_conditions:\n> raise AssertionError(\n f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n )\nE AssertionError: Missing true conditions: {'WorkloadsReady', 'RouterReady', 'Ready'}, expected {'WorkloadsReady', 'RouterReady', 'Ready'}, got [{'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'status': 'Unknown', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-20T11:09:53Z', 'severity': 'Info', 'status': 'True', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-20T11:09:39Z', 'message': 'failed to reconcile main workload scaling: failed to reconcile main VA: failed to get v1alpha1.VariantAutoscaling e2e-test-llm-autoscaling-keda-lws-e541a132/autoscale-keda-lws-kserve-va: no matches for kind \"VariantAutoscaling\" in version \"llmd.ai/v1alpha1\"', 'reason': 'ScalingCRDNotFound', 'status': 'False', 'type': 'WorkloadsReady'}]\n\nllmisvc/test_llm_inference_service.py:1250: AssertionError"}, "teardown": {"duration": 0.038249091998295626, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "custom_gateway", "__wrapped__", "pytestmark", "router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 2.462184274001629, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 252.15583707800033, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.06658222099940758, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-no-scheduler-workload-single-cpu-model-fb-opt-125m]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-no-scheduler-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "no_scheduler", "__wrapped__", "pytestmark", "router-no-scheduler-workload-single-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.4862667010020232, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 155.4959888730009, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.039085079999495065, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_multi_node-router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "cluster_cpu", "cluster_multi_node", "__wrapped__", "pytestmark", "router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.29540049400020507, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 199.8686813259992, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.03942865499993786, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-inline-config-workload-llmd-simulator]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-inline-config-workload-llmd-simulator]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-inline-config-workload-llmd-simulator", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.28572913000243716, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 61.80590932099949, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04196009500083164, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator-model-qwen2.5-0.5b]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator-model-qwen2.5-0.5b]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-model-qwen2.5-0.5b", "llminferenceservice", "llmisvc_core", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.25895707800009404, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 73.01008545500008, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.04654867199860746, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-configmap-ref-workload-llmd-simulator]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-configmap-ref-workload-llmd-simulator]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-configmap-ref-workload-llmd-simulator", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.34403080600168323, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 59.31467214899749, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.06645689699871582, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-replicas-workload-llmd-simulator]", "lineno": 242, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-replicas-workload-llmd-simulator]", "parametrize", "asyncio", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-replicas-workload-llmd-simulator", "llminferenceservice", "llmisvc_core", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.26284073700298904, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 62.107082483998965, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.050591971001267666, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}], "warnings": [{"message": "The event_loop fixture provided by pytest-asyncio has been redefined in\n/workspace/source/test/e2e/conftest.py:43\nReplacing the event_loop fixture with a custom implementation is deprecated\nand will lead to errors in the future.\nIf you want to request an asyncio event loop with a scope other than function\nscope, use the \"scope\" argument to the asyncio mark when marking the tests.\nIf you want to return different types of event loops, use the event_loop_policy\nfixture.\n", "category": "DeprecationWarning", "when": "runtest", "filename": "/workspace/source/python/kserve/.venv/lib64/python3.11/site-packages/pytest_asyncio/plugin.py", "lineno": 761}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The event_loop fixture provided by pytest-asyncio has been redefined in\n/workspace/source/test/e2e/conftest.py:43\nReplacing the event_loop fixture with a custom implementation is deprecated\nand will lead to errors in the future.\nIf you want to request an asyncio event loop with a scope other than function\nscope, use the \"scope\" argument to the asyncio mark when marking the tests.\nIf you want to return different types of event loops, use the event_loop_policy\nfixture.\n", "category": "DeprecationWarning", "when": "runtest", "filename": "/workspace/source/python/kserve/.venv/lib64/python3.11/site-packages/pytest_asyncio/plugin.py", "lineno": 761}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_flow_control.py", "lineno": 47}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_flow_control.py", "lineno": 47}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service_stop.py", "lineno": 40}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_tls.py", "lineno": 92}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}, {"message": "The test is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 243}]}