('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')file /workspace/source/test/e2e/graph/test_inference_graph.py, line 39 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_inference_graph(rest_v1_client, network_layer): logger.info("Starting test test_inference_graph") sklearn_name_1 = "isvc-sklearn-graph-1" sklearn_name_2 = "isvc-sklearn-graph-2" xgb_name = "isvc-xgboost-graph" graph_name = "model-chainer" sklearn_predictor_1 = V1beta1PredictorSpec( min_replicas=1, sklearn=V1beta1SKLearnSpec( storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), args=["--model_name=sklearn"], ), ) sklearn_predictor_2 = V1beta1PredictorSpec( min_replicas=1, sklearn=V1beta1SKLearnSpec( storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), args=["--model_name", "iris"], ), ) sklearn_isvc_1 = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta( name=sklearn_name_1, namespace=KSERVE_TEST_NAMESPACE ), spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor_1), ) sklearn_isvc_2 = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta( name=sklearn_name_2, namespace=KSERVE_TEST_NAMESPACE ), spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor_2), ) xgb_predictor = V1beta1PredictorSpec( min_replicas=1, xgboost=V1beta1XGBoostSpec( storage_uri="gs://kfserving-examples/models/xgboost/1.5/model", resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), ), ) xgb_isvc = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta(name=xgb_name, namespace=KSERVE_TEST_NAMESPACE), spec=V1beta1InferenceServiceSpec(predictor=xgb_predictor), ) nodes = { "root": V1alpha1InferenceRouter( router_type="Sequence", steps=[ V1alpha1InferenceStep( service_name=sklearn_name_1, dependency="Hard", ), V1alpha1InferenceStep( service_name=xgb_name, data="$request", dependency="Hard", ), V1alpha1InferenceStep( service_name=sklearn_name_2, data="$request", dependency="Hard", ), ], ) } graph_spec = V1alpha1InferenceGraphSpec( nodes=nodes, ) ig = V1alpha1InferenceGraph( api_version=constants.KSERVE_V1ALPHA1, kind=constants.KSERVE_KIND_INFERENCEGRAPH, metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE), spec=graph_spec, ) kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(sklearn_isvc_1) kserve_client.create(xgb_isvc) kserve_client.create(sklearn_isvc_2) kserve_client.wait_isvc_ready(sklearn_name_1, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(xgb_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(sklearn_name_2, namespace=KSERVE_TEST_NAMESPACE) kserve_client.create_inference_graph(ig) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) res = await predict_ig( rest_v1_client, graph_name, os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, "iris_input.json"), network_layer=network_layer, ) assert res["predictions"] == [1, 1] kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(sklearn_name_1, KSERVE_TEST_NAMESPACE) kserve_client.delete(xgb_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(sklearn_name_2, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 547 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario5(rest_v1_client, network_layer): """ Scenario: Switch graph where a match would happen for error node and then error would return but IG will continue execution and call the next step in the flow as error step will be a soft dependency. Expectation: IG will return response of success_isvc. """ logger.info("Starting test test_ig_scenario5") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["switch-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_switch_scenario_5.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) response = await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_error_picker_input.json" ), network_layer=network_layer, ) assert response == {"predictions": [{"message": "SUCCESS"}]} kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 226 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario1(rest_v1_client, network_layer): """ Scenario: Sequence graph with 2 steps that are both soft dependencies. success_isvc(soft) -> error_isvc (soft) We are not marking steps as soft or hard explicitly so this will test that default behavior of steps being soft is as expected. Expectation: IG will return response of error_isvc and predict_ig will raise exception :return: """ logger.info("Starting test test_ig_scenario1") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") # Create graph graph_name = "-".join(["sequence-graph", suffix]) nodes = { "root": V1alpha1InferenceRouter( router_type="Sequence", steps=[ V1alpha1InferenceStep( service_name=success_isvc_name, ), V1alpha1InferenceStep( service_name=error_isvc_name, data="$request", ), ], ) } graph_spec = V1alpha1InferenceGraphSpec( nodes=nodes, ) ig = V1alpha1InferenceGraph( api_version=constants.KSERVE_V1ALPHA1, kind=constants.KSERVE_KIND_INFERENCEGRAPH, metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE), spec=graph_spec, ) kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.create_inference_graph(ig) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) with pytest.raises(HTTPStatusError) as exc_info: await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "custom_predictor_input.json" ), network_layer=network_layer, ) assert exc_info.value.response.json() == {"detail": "Intentional 404 code"} assert exc_info.value.response.status_code == 404 kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 612 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario6(rest_v1_client, network_layer): """ Scenario: Switch graph where a match would happen for error node and then error would return and IG will NOT continue execution and call the next step in the flow as error step will be a HARD dependency. Expectation: IG will return response of success_isvc. """ logger.info("Starting test test_ig_scenario6") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["switch-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_switch_scenario_6.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) with pytest.raises(HTTPStatusError) as exc_info: await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_error_picker_input.json" ), network_layer=network_layer, ) assert exc_info.value.response.json() == {"detail": "Intentional 404 code"} assert exc_info.value.response.status_code == 404 kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 304 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario2(rest_v1_client, network_layer): """ Scenario: Sequence graph with 2 steps that are both soft dependencies. error_isvc (soft) -> success_isvc(soft) Expectation: IG will return response of success_isvc and predict_ig will not raise any exception :return: """ logger.info("Starting test test_ig_scenario2") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") # Create graph graph_name = "-".join(["sequence-graph", suffix]) nodes = { "root": V1alpha1InferenceRouter( router_type="Sequence", steps=[ V1alpha1InferenceStep( service_name=error_isvc_name, ), V1alpha1InferenceStep( service_name=success_isvc_name, data="$request", ), ], ) } graph_spec = V1alpha1InferenceGraphSpec( nodes=nodes, ) ig = V1alpha1InferenceGraph( api_version=constants.KSERVE_V1ALPHA1, kind=constants.KSERVE_KIND_INFERENCEGRAPH, metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE), spec=graph_spec, ) kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.create_inference_graph(ig) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) response = await predict_ig( rest_v1_client, graph_name, os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, "custom_predictor_input.json"), network_layer=network_layer, ) assert response == {"predictions": [{"message": "SUCCESS"}]} kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 680 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario7(rest_v1_client, network_layer): """ Scenario: Ensemble graph with 2 steps, where both the steps are soft deps. Expectation: IG will return combined response of both the steps. """ logger.info("Starting test test_ig_scenario7") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["ensemble-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_ensemble_scenario_7.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) response = await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_success_picker_input.json" ), network_layer=network_layer, ) assert response == { "rootStep1": {"predictions": [{"message": "SUCCESS"}]}, "rootStep2": {"detail": "Intentional 404 code"}, } kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 375 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario3(rest_v1_client, network_layer): """ Scenario: Sequence graph with 2 steps - first is hard (and returns non-200) and second is soft dependency. error_isvc(hard) -> success_isvc (soft) Expectation: IG will return response of error_isvc and predict_ig will raise exception """ logger.info("Starting test test_ig_scenario3") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["sequence-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_seq_scenario_3.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) with pytest.raises(HTTPStatusError) as exc_info: await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "custom_predictor_input.json" ), network_layer=network_layer, ) assert exc_info.value.response.json() == {"detail": "Intentional 404 code"} assert exc_info.value.response.status_code == 404 kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 748 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario8(rest_v1_client, network_layer): """ Scenario: Ensemble graph with 3 steps, where 2 steps are soft and 1 step is hard and returns non-200 Expectation: Since HARD step will return non-200, so IG will return that step's output as IG's output """ logger.info("Starting test test_ig_scenario8") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["ensemble-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_ensemble_scenario_8.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) with pytest.raises(HTTPStatusError) as exc_info: await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_success_picker_input.json" ), network_layer=network_layer, ) assert exc_info.value.response.json() == {"detail": "Intentional 404 code"} kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 444 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario4(rest_v1_client, network_layer): """ Scenario: Switch graph with 1 step as hard dependency and other one as soft dependency. Will be testing 3 cases in this test case: Expectation: Case 1. IG will return response of error_isvc when condition for that step matches Case 2. IG will return response of success_isvc when condition for that step matches Case 3. IG will return 404 with error message when no condition matches { "error": "Failed to process request", "cause": "None of the routes matched with the switch condition", } """ logger.info("Starting test test_ig_scenario4") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["switch-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_switch_scenario_4.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) # Case 1 with pytest.raises(HTTPStatusError) as exc_info: await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_error_picker_input.json" ), network_layer=network_layer, ) assert exc_info.value.response.json() == {"detail": "Intentional 404 code"} assert exc_info.value.response.status_code == 404 # Case 2 response = await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_success_picker_input.json" ), network_layer=network_layer, ) assert response == {"predictions": [{"message": "SUCCESS"}]} # Case 3 with pytest.raises(HTTPStatusError) as exc_info: await predict_ig( rest_v1_client, graph_name, os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_no_match_input.json" ), network_layer=network_layer, ) assert exc_info.value.response.json() == { "error": "Failed to process request", "cause": "None of the routes matched with the switch condition", } assert exc_info.value.response.status_code == 404 kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/graph/test_inference_graph.py, line 815 @pytest.mark.graph @pytest.mark.kourier @pytest.mark.asyncio(scope="session") async def test_ig_scenario9(rest_v1_client, network_layer): """ Scenario: Splitter graph where a match would happen for error node and then error would return but IG will continue execution and call the next step in the flow as error step will be a soft dependency. Expectation: IG will return response of success_isvc. """ logger.info("Starting test test_ig_scenario9") suffix = str(uuid.uuid4())[1:6] success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test( suffix ) logger.info(f"success_isvc_name is {success_isvc_name}") logger.info(f"error_isvc_name is {error_isvc_name}") kserve_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) kserve_client.create(success_isvc) kserve_client.create(error_isvc) # Create graph graph_name = "-".join(["splitter-graph", suffix]) # Because we run from test/e2e location in run-e2e-tests.sh deployment_yaml_path = os.path.join( IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_switch_scenario_9.yaml" ) # Read YAML file with open(deployment_yaml_path, "r") as stream: file_content = stream.read() resource_template = Template(file_content) substitutions = { "graph_name": graph_name, "error_404_isvc_id": error_isvc_name, "success_200_isvc_id": success_isvc_name, } resource_body_after_rendering = yaml.safe_load( resource_template.render(substitutions) ) kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE) kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE) create_ig_using_custom_object_api(resource_body_after_rendering) kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE) response = await predict_ig( rest_v1_client, graph_name, os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, "iris_input.json"), network_layer=network_layer, ) assert response == {"predictions": [{"message": "SUCCESS"}]} kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(success_isvc_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(error_isvc_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::<event_loop>, graph/test_inference_graph.py::<event_loop>, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121