('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')file /workspace/source/test/e2e/batcher/test_batcher.py, line 31 @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_batcher(rest_v1_client, network_layer): service_name = "isvc-sklearn-batcher" predictor = V1beta1PredictorSpec( batcher=V1beta1Batcher( max_batch_size=32, max_latency=5000, ), min_replicas=1, sklearn=V1beta1SKLearnSpec( storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), ), ) isvc = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta( name=service_name, namespace=KSERVE_TEST_NAMESPACE, labels={ constants.KSERVE_LABEL_NETWORKING_VISIBILITY: constants.KSERVE_LABEL_NETWORKING_VISIBILITY_EXPOSED, }, ), spec=V1beta1InferenceServiceSpec(predictor=predictor), ) kserve_client.create(isvc) try: kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) except RuntimeError as e: print( kserve_client.api_instance.get_namespaced_custom_object( "serving.knative.dev", "v1", KSERVE_TEST_NAMESPACE, "services", service_name + "-predictor", ) ) pods = kserve_client.core_api.list_namespaced_pod( KSERVE_TEST_NAMESPACE, label_selector="serving.kserve.io/inferenceservice={}".format(service_name), ) for pod in pods.items: print(pod) raise e results = await predict_isvc( rest_v1_client, service_name, "./data/iris_batch_input.json", is_batch=True, network_layer=network_layer, ) assert all(x == results[0] for x in results) kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, batcher/__init__.py::<event_loop>, batcher/test_batcher.py::<event_loop>, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/batcher/test_batcher_custom_port.py, line 33 @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_batcher_custom_port(rest_v1_client, network_layer): service_name = "isvc-sklearn-batcher-custom" predictor = V1beta1PredictorSpec( batcher=V1beta1Batcher( max_batch_size=32, max_latency=5000, ), min_replicas=1, sklearn=V1beta1SKLearnSpec( args=["--http_port=5000"], storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", resources=V1ResourceRequirements( requests={"cpu": "50m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), ports=[V1ContainerPort(container_port=5000, protocol="TCP")], ), ) isvc = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta( name=service_name, namespace=KSERVE_TEST_NAMESPACE, labels={ constants.KSERVE_LABEL_NETWORKING_VISIBILITY: constants.KSERVE_LABEL_NETWORKING_VISIBILITY_EXPOSED, }, ), spec=V1beta1InferenceServiceSpec(predictor=predictor), ) kserve_client.create(isvc) try: kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) except RuntimeError as e: print( kserve_client.api_instance.get_namespaced_custom_object( "serving.knative.dev", "v1", KSERVE_TEST_NAMESPACE, "services", service_name + "-predictor", ) ) pods = kserve_client.core_api.list_namespaced_pod( KSERVE_TEST_NAMESPACE, label_selector="serving.kserve.io/inferenceservice={}".format(service_name), ) for pod in pods.items: print(pod) raise e results = await predict_isvc( rest_v1_client, service_name, "./data/iris_batch_input.json", is_batch=True, network_layer=network_layer, ) assert all(x == results[0] for x in results) kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: __init__.py::<event_loop>, _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, batcher/test_batcher_custom_port.py::<event_loop>, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121/workspace/source/test/e2e/custom/test_custom_model_grpc.py:42: Not testable in ODH at the moment/workspace/source/test/e2e/custom/test_ray.py:30: Not testable in ODH at the momentfile /workspace/source/test/e2e/logger/test_logger.py, line 34 @pytest.mark.predictor @pytest.mark.path_based_routing @pytest.mark.asyncio(scope="session") async def test_kserve_logger(rest_v1_client, network_layer): msg_dumper = "message-dumper" predictor = V1beta1PredictorSpec( min_replicas=1, containers=[ V1Container( name="kserve-container", image="gcr.io/knative-releases/knative.dev/eventing-contrib/cmd/event_display", resources=V1ResourceRequirements( requests={"cpu": "10m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), ) ], ) isvc = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta( name=msg_dumper, namespace=KSERVE_TEST_NAMESPACE, labels={ constants.KSERVE_LABEL_NETWORKING_VISIBILITY: constants.KSERVE_LABEL_NETWORKING_VISIBILITY_EXPOSED, }, ), spec=V1beta1InferenceServiceSpec(predictor=predictor), ) kserve_client.create(isvc) kserve_client.wait_isvc_ready(msg_dumper, namespace=KSERVE_TEST_NAMESPACE) service_name = "isvc-logger" predictor = V1beta1PredictorSpec( min_replicas=1, logger=V1beta1LoggerSpec( mode="all", url=f"http://{msg_dumper}-predictor." + KSERVE_TEST_NAMESPACE + ".svc.cluster.local", ), sklearn=V1beta1SKLearnSpec( storage_uri="gs://kfserving-examples/models/sklearn/1.0/model", resources=V1ResourceRequirements( requests={"cpu": "10m", "memory": "128Mi"}, limits={"cpu": "100m", "memory": "256Mi"}, ), ), ) isvc = V1beta1InferenceService( api_version=constants.KSERVE_V1BETA1, kind=constants.KSERVE_KIND_INFERENCESERVICE, metadata=client.V1ObjectMeta( name=service_name, namespace=KSERVE_TEST_NAMESPACE, labels={ constants.KSERVE_LABEL_NETWORKING_VISIBILITY: constants.KSERVE_LABEL_NETWORKING_VISIBILITY_EXPOSED, }, ), spec=V1beta1InferenceServiceSpec(predictor=predictor), ) kserve_client.create(isvc) try: kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE) except RuntimeError: pods = kserve_client.core_api.list_namespaced_pod( KSERVE_TEST_NAMESPACE, label_selector="serving.kserve.io/inferenceservice={}".format(service_name), ) for pod in pods.items: print(pod) res = await predict_isvc( rest_v1_client, service_name, "./data/iris_input.json", network_layer=network_layer, ) assert res["predictions"] == [1, 1] pods = kserve_client.core_api.list_namespaced_pod( KSERVE_TEST_NAMESPACE, label_selector="serving.kserve.io/inferenceservice={}".format(msg_dumper), ) await asyncio.sleep(5) log = "" for pod in pods.items: log += kserve_client.core_api.read_namespaced_pod_log( name=pod.metadata.name, namespace=pod.metadata.namespace, container="kserve-container", ) print(log) assert "org.kubeflow.serving.inference.request" in log assert "org.kubeflow.serving.inference.response" in log kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE) kserve_client.delete(msg_dumper, KSERVE_TEST_NAMESPACE) file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, logger/__init__.py::<event_loop>, logger/test_logger.py::<event_loop>, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/predictor/test_autogluon.py, line 58 @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_runtime_kserve_v1(rest_v1_client): service_name = "isvc-autogluon-v1" predictor = _create_predictor(service_name) response = await deploy_and_predict( service_name, predictor, rest_v1_client, "./data/autogluon_titanic_input.json", ) assert "predictions" in response assert len(response["predictions"]) > 0 file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _cleanup_orphaned_predictor_isvcs, _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, predictor/__init__.py::<event_loop>, predictor/test_autogluon.py::<event_loop>, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121file /workspace/source/test/e2e/predictor/test_autogluon.py, line 73 @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_runtime_kserve_v2(rest_v2_client): service_name = "isvc-autogluon-v2" predictor = _create_predictor(service_name, protocol_version="v2") response = await deploy_and_predict( service_name, predictor, rest_v2_client, "./data/autogluon_titanic_input_v2.json", ) assert len(response.outputs) > 0 assert len(response.outputs[0].data) > 0 file /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121 @pytest.fixture(autouse=True) def ensure_gateway_proxy_memory(test_case): E fixture 'test_case' not found > available fixtures: _cleanup_orphaned_predictor_isvcs, _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, predictor/__init__.py::<event_loop>, predictor/test_autogluon.py::<event_loop>, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id > use 'pytest --fixtures [testpath]' for help on them. /workspace/source/test/e2e/common/gateway_proxy_istio.py:121