{"created": 1783371548.1091664, "duration": 2.676764965057373, "exitcode": 2, "root": "/workspace/source/test/e2e", "environment": {}, "summary": {"error": 10, "total": 10, "collected": 11}, "collectors": [{"nodeid": "explainer/test_art_explainer.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')"}, {"nodeid": "predictor/test_grpc.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')"}, {"nodeid": "predictor/test_torchserve.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')"}], "tests": [{"nodeid": "graph/test_inference_graph.py::test_inference_graph", "lineno": 38, "outcome": "error", "keywords": ["test_inference_graph", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.0037673309998353943, "outcome": "failed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 39\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_inference_graph(rest_v1_client, network_layer):\n logger.info(\"Starting test test_inference_graph\")\n sklearn_name_1 = \"isvc-sklearn-graph-1\"\n sklearn_name_2 = \"isvc-sklearn-graph-2\"\n xgb_name = \"isvc-xgboost-graph\"\n graph_name = \"model-chainer\"\n\n sklearn_predictor_1 = V1beta1PredictorSpec(\n min_replicas=1,\n sklearn=V1beta1SKLearnSpec(\n storage_uri=\"gs://kfserving-examples/models/sklearn/1.0/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n args=[\"--model_name=sklearn\"],\n ),\n )\n sklearn_predictor_2 = V1beta1PredictorSpec(\n min_replicas=1,\n sklearn=V1beta1SKLearnSpec(\n storage_uri=\"gs://kfserving-examples/models/sklearn/1.0/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n args=[\"--model_name\", \"iris\"],\n ),\n )\n sklearn_isvc_1 = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=sklearn_name_1, namespace=KSERVE_TEST_NAMESPACE\n ),\n spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor_1),\n )\n sklearn_isvc_2 = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=sklearn_name_2, namespace=KSERVE_TEST_NAMESPACE\n ),\n spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor_2),\n )\n\n xgb_predictor = V1beta1PredictorSpec(\n min_replicas=1,\n xgboost=V1beta1XGBoostSpec(\n storage_uri=\"gs://kfserving-examples/models/xgboost/1.5/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n xgb_isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(name=xgb_name, namespace=KSERVE_TEST_NAMESPACE),\n spec=V1beta1InferenceServiceSpec(predictor=xgb_predictor),\n )\n\n nodes = {\n \"root\": V1alpha1InferenceRouter(\n router_type=\"Sequence\",\n steps=[\n V1alpha1InferenceStep(\n service_name=sklearn_name_1,\n dependency=\"Hard\",\n ),\n V1alpha1InferenceStep(\n service_name=xgb_name,\n data=\"$request\",\n dependency=\"Hard\",\n ),\n V1alpha1InferenceStep(\n service_name=sklearn_name_2,\n data=\"$request\",\n dependency=\"Hard\",\n ),\n ],\n )\n }\n graph_spec = V1alpha1InferenceGraphSpec(\n nodes=nodes,\n )\n ig = V1alpha1InferenceGraph(\n api_version=constants.KSERVE_V1ALPHA1,\n kind=constants.KSERVE_KIND_INFERENCEGRAPH,\n metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE),\n spec=graph_spec,\n )\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(sklearn_isvc_1)\n kserve_client.create(xgb_isvc)\n kserve_client.create(sklearn_isvc_2)\n kserve_client.wait_isvc_ready(sklearn_name_1, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(xgb_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(sklearn_name_2, namespace=KSERVE_TEST_NAMESPACE)\n\n kserve_client.create_inference_graph(ig)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n res = await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, \"iris_input.json\"),\n network_layer=network_layer,\n )\n assert res[\"predictions\"] == [1, 1]\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(sklearn_name_1, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(xgb_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(sklearn_name_2, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.0001568509978824295, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario5", "lineno": 546, "outcome": "error", "keywords": ["test_ig_scenario5", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.00532411700260127, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 547\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario5(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Switch graph where a match would happen for error node and then error would return but IG will continue\n execution and call the next step in the flow as error step will be a soft dependency.\n Expectation: IG will return response of success_isvc.\n \"\"\"\n logger.info(\"Starting test test_ig_scenario5\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"switch-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_switch_scenario_5.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n response = await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_error_picker_input.json\"\n ),\n network_layer=network_layer,\n )\n assert response == {\"predictions\": [{\"message\": \"SUCCESS\"}]}\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00017573300283402205, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario1", "lineno": 225, "outcome": "error", "keywords": ["test_ig_scenario1", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.0002449690000503324, "outcome": "failed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 226\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario1(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Sequence graph with 2 steps that are both soft dependencies.\n success_isvc(soft) -> error_isvc (soft)\n\n We are not marking steps as soft or hard explicitly so this will test that default behavior of steps being soft\n is as expected.\n Expectation: IG will return response of error_isvc and predict_ig will raise exception\n :return:\n \"\"\"\n\n logger.info(\"Starting test test_ig_scenario1\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n # Create graph\n graph_name = \"-\".join([\"sequence-graph\", suffix])\n\n nodes = {\n \"root\": V1alpha1InferenceRouter(\n router_type=\"Sequence\",\n steps=[\n V1alpha1InferenceStep(\n service_name=success_isvc_name,\n ),\n V1alpha1InferenceStep(\n service_name=error_isvc_name,\n data=\"$request\",\n ),\n ],\n )\n }\n graph_spec = V1alpha1InferenceGraphSpec(\n nodes=nodes,\n )\n ig = V1alpha1InferenceGraph(\n api_version=constants.KSERVE_V1ALPHA1,\n kind=constants.KSERVE_KIND_INFERENCEGRAPH,\n metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE),\n spec=graph_spec,\n )\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n kserve_client.create_inference_graph(ig)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n with pytest.raises(HTTPStatusError) as exc_info:\n await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"custom_predictor_input.json\"\n ),\n network_layer=network_layer,\n )\n\n assert exc_info.value.response.json() == {\"detail\": \"Intentional 404 code\"}\n assert exc_info.value.response.status_code == 404\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00015900200378382578, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario6", "lineno": 611, "outcome": "error", "keywords": ["test_ig_scenario6", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.00023851800506236032, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 612\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario6(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Switch graph where a match would happen for error node and then error would return and IG will NOT\n continue execution and call the next step in the flow as error step will be a HARD dependency.\n Expectation: IG will return response of success_isvc.\n \"\"\"\n logger.info(\"Starting test test_ig_scenario6\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"switch-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_switch_scenario_6.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n with pytest.raises(HTTPStatusError) as exc_info:\n await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_error_picker_input.json\"\n ),\n network_layer=network_layer,\n )\n\n assert exc_info.value.response.json() == {\"detail\": \"Intentional 404 code\"}\n assert exc_info.value.response.status_code == 404\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00013256000238470733, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario2", "lineno": 303, "outcome": "error", "keywords": ["test_ig_scenario2", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.0002169860017602332, "outcome": "failed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 304\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario2(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Sequence graph with 2 steps that are both soft dependencies.\n error_isvc (soft) -> success_isvc(soft)\n\n Expectation: IG will return response of success_isvc and predict_ig will not raise any exception\n :return:\n \"\"\"\n\n logger.info(\"Starting test test_ig_scenario2\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n # Create graph\n graph_name = \"-\".join([\"sequence-graph\", suffix])\n\n nodes = {\n \"root\": V1alpha1InferenceRouter(\n router_type=\"Sequence\",\n steps=[\n V1alpha1InferenceStep(\n service_name=error_isvc_name,\n ),\n V1alpha1InferenceStep(\n service_name=success_isvc_name,\n data=\"$request\",\n ),\n ],\n )\n }\n graph_spec = V1alpha1InferenceGraphSpec(\n nodes=nodes,\n )\n ig = V1alpha1InferenceGraph(\n api_version=constants.KSERVE_V1ALPHA1,\n kind=constants.KSERVE_KIND_INFERENCEGRAPH,\n metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE),\n spec=graph_spec,\n )\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n kserve_client.create_inference_graph(ig)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n response = await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, \"custom_predictor_input.json\"),\n network_layer=network_layer,\n )\n assert response == {\"predictions\": [{\"message\": \"SUCCESS\"}]}\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00013489900447893888, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario7", "lineno": 679, "outcome": "error", "keywords": ["test_ig_scenario7", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.00021037599799456075, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 680\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario7(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Ensemble graph with 2 steps, where both the steps are soft deps.\n\n Expectation: IG will return combined response of both the steps.\n \"\"\"\n logger.info(\"Starting test test_ig_scenario7\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"ensemble-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_ensemble_scenario_7.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n response = await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_success_picker_input.json\"\n ),\n network_layer=network_layer,\n )\n assert response == {\n \"rootStep1\": {\"predictions\": [{\"message\": \"SUCCESS\"}]},\n \"rootStep2\": {\"detail\": \"Intentional 404 code\"},\n }\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.0001329689985141158, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario3", "lineno": 374, "outcome": "error", "keywords": ["test_ig_scenario3", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.0002551790021243505, "outcome": "failed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 375\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario3(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Sequence graph with 2 steps - first is hard (and returns non-200) and second is soft dependency.\n error_isvc(hard) -> success_isvc (soft)\n\n Expectation: IG will return response of error_isvc and predict_ig will raise exception\n \"\"\"\n logger.info(\"Starting test test_ig_scenario3\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"sequence-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_seq_scenario_3.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n with pytest.raises(HTTPStatusError) as exc_info:\n await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"custom_predictor_input.json\"\n ),\n network_layer=network_layer,\n )\n\n assert exc_info.value.response.json() == {\"detail\": \"Intentional 404 code\"}\n assert exc_info.value.response.status_code == 404\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00013285000022733584, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario8", "lineno": 747, "outcome": "error", "keywords": ["test_ig_scenario8", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.000213225997867994, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 748\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario8(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Ensemble graph with 3 steps, where 2 steps are soft and 1 step is hard and returns non-200\n\n Expectation: Since HARD step will return non-200, so IG will return that step's output as IG's output\n \"\"\"\n logger.info(\"Starting test test_ig_scenario8\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"ensemble-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_ensemble_scenario_8.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n with pytest.raises(HTTPStatusError) as exc_info:\n await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_success_picker_input.json\"\n ),\n network_layer=network_layer,\n )\n\n assert exc_info.value.response.json() == {\"detail\": \"Intentional 404 code\"}\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00013017000310355797, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario4", "lineno": 443, "outcome": "error", "keywords": ["test_ig_scenario4", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.00020756500452989712, "outcome": "failed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 444\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario4(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Switch graph with 1 step as hard dependency and other one as soft dependency.\n Will be testing 3 cases in this test case:\n Expectation:\n Case 1. IG will return response of error_isvc when condition for that step matches\n Case 2. IG will return response of success_isvc when condition for that step matches\n Case 3. IG will return 404 with error message when no condition matches\n {\n \"error\": \"Failed to process request\",\n \"cause\": \"None of the routes matched with the switch condition\",\n }\n \"\"\"\n logger.info(\"Starting test test_ig_scenario4\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"switch-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_switch_scenario_4.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n # Case 1\n with pytest.raises(HTTPStatusError) as exc_info:\n await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_error_picker_input.json\"\n ),\n network_layer=network_layer,\n )\n\n assert exc_info.value.response.json() == {\"detail\": \"Intentional 404 code\"}\n assert exc_info.value.response.status_code == 404\n\n # Case 2\n response = await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_success_picker_input.json\"\n ),\n network_layer=network_layer,\n )\n assert response == {\"predictions\": [{\"message\": \"SUCCESS\"}]}\n\n # Case 3\n with pytest.raises(HTTPStatusError) as exc_info:\n await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"switch_call_no_match_input.json\"\n ),\n network_layer=network_layer,\n )\n\n assert exc_info.value.response.json() == {\n \"error\": \"Failed to process request\",\n \"cause\": \"None of the routes matched with the switch condition\",\n }\n assert exc_info.value.response.status_code == 404\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.1248596009972971, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_ig_scenario9", "lineno": 814, "outcome": "error", "keywords": ["test_ig_scenario9", "asyncio", "kourier", "graph", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.00022844700288260356, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 815\n @pytest.mark.graph\n @pytest.mark.kourier\n @pytest.mark.asyncio(scope=\"session\")\n async def test_ig_scenario9(rest_v1_client, network_layer):\n \"\"\"\n Scenario: Splitter graph where a match would happen for error node and then error would return but IG will continue\n execution and call the next step in the flow as error step will be a soft dependency.\n Expectation: IG will return response of success_isvc.\n \"\"\"\n logger.info(\"Starting test test_ig_scenario9\")\n suffix = str(uuid.uuid4())[1:6]\n success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(\n suffix\n )\n logger.info(f\"success_isvc_name is {success_isvc_name}\")\n logger.info(f\"error_isvc_name is {error_isvc_name}\")\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(success_isvc)\n kserve_client.create(error_isvc)\n\n # Create graph\n graph_name = \"-\".join([\"splitter-graph\", suffix])\n\n # Because we run from test/e2e location in run-e2e-tests.sh\n deployment_yaml_path = os.path.join(\n IG_TEST_RESOURCES_BASE_LOCATION, \"ig_test_switch_scenario_9.yaml\"\n )\n\n # Read YAML file\n with open(deployment_yaml_path, \"r\") as stream:\n file_content = stream.read()\n resource_template = Template(file_content)\n substitutions = {\n \"graph_name\": graph_name,\n \"error_404_isvc_id\": error_isvc_name,\n \"success_200_isvc_id\": success_isvc_name,\n }\n resource_body_after_rendering = yaml.safe_load(\n resource_template.render(substitutions)\n )\n\n kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)\n\n create_ig_using_custom_object_api(resource_body_after_rendering)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n response = await predict_ig(\n rest_v1_client,\n graph_name,\n os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, \"iris_input.json\"),\n network_layer=network_layer,\n )\n assert response == {\"predictions\": [{\"message\": \"SUCCESS\"}]}\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00014063099661143497, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}]}