{"created": 1783371457.2174168, "duration": 2.603208065032959, "exitcode": 2, "root": "/workspace/source/test/e2e", "environment": {}, "summary": {"error": 5, "skipped": 1, "total": 6, "collected": 15}, "collectors": [{"nodeid": "explainer/test_art_explainer.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')"}, {"nodeid": "predictor/test_grpc.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')"}, {"nodeid": "predictor/test_torchserve.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')"}], "tests": [{"nodeid": "batcher/test_raw_batcher.py::test_batcher_raw", "lineno": 32, "outcome": "error", "keywords": ["test_batcher_raw", "asyncio", "raw", "pytestmark", "test_raw_batcher.py", "batcher/__init__.py", "e2e"], "setup": {"duration": 0.004589128999214154, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/batcher/test_raw_batcher.py, line 33\n @pytest.mark.raw\n @pytest.mark.asyncio(scope=\"session\")\n async def test_batcher_raw(rest_v1_client, network_layer):\n suffix = str(uuid.uuid4())[1:6]\n service_name = \"isvc-raw-sklearn-batcher-\" + suffix\n\n annotations = dict()\n annotations[\"serving.kserve.io/deploymentMode\"] = \"Standard\"\n\n labels = dict()\n labels[\"networking.kserve.io/visibility\"] = \"exposed\"\n\n predictor = V1beta1PredictorSpec(\n batcher=V1beta1Batcher(\n max_batch_size=32,\n max_latency=5000,\n ),\n min_replicas=1,\n sklearn=V1beta1SKLearnSpec(\n storage_uri=\"gs://kfserving-examples/models/sklearn/1.0/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n\n isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=service_name,\n namespace=KSERVE_TEST_NAMESPACE,\n annotations=annotations,\n labels=labels,\n ),\n spec=V1beta1InferenceServiceSpec(predictor=predictor),\n )\n kserve_client.create(isvc)\n try:\n kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)\n except RuntimeError as e:\n print(\n kserve_client.api_instance.get_namespaced_custom_object(\n \"serving.knative.dev\",\n \"v1\",\n KSERVE_TEST_NAMESPACE,\n \"services\",\n service_name + \"-predictor\",\n )\n )\n pods = kserve_client.core_api.list_namespaced_pod(\n KSERVE_TEST_NAMESPACE,\n label_selector=\"serving.kserve.io/inferenceservice={}\".format(service_name),\n )\n for pod in pods.items:\n print(pod)\n raise e\n results = await predict_isvc(\n rest_v1_client,\n service_name,\n \"./data/iris_batch_input.json\",\n is_batch=True,\n network_layer=network_layer,\n )\n assert all(x == results[0] for x in results)\n kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, batcher/test_raw_batcher.py::, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.0001767450012266636, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "custom/test_custom_model_grpc.py::test_predictor_grpc_with_transformer_grpc_raw", "lineno": 378, "outcome": "skipped", "keywords": ["test_predictor_grpc_with_transformer_grpc_raw", "asyncio", "raw", "skip", "pytestmark", "test_custom_model_grpc.py", "custom/__init__.py", "e2e"], "setup": {"duration": 0.0001557040013722144, "outcome": "skipped", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n('/workspace/source/test/e2e/custom/test_custom_model_grpc.py', 379, 'Skipped: Not testable in ODH at the moment')"}, "teardown": {"duration": 0.00011443299445090815, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_inference_graph_raw_mode", "lineno": 943, "outcome": "error", "keywords": ["test_inference_graph_raw_mode", "asyncio", "raw", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.000401960001909174, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 944\n @pytest.mark.raw\n @pytest.mark.asyncio(scope=\"session\")\n async def test_inference_graph_raw_mode(rest_v1_client, network_layer):\n logger.info(\"Starting test test_inference_graph_raw_mode\")\n suffix = str(uuid.uuid4())[1:6]\n sklearn_name = \"isvc-sklearn-graph-raw-\" + suffix\n xgb_name = \"isvc-xgboost-graph-raw-\" + suffix\n graph_name = \"model-chainer-raw-\" + suffix\n\n annotations = dict()\n annotations[\"serving.kserve.io/deploymentMode\"] = \"Standard\"\n labels = dict()\n labels[\"networking.kserve.io/visibility\"] = \"exposed\"\n\n sklearn_predictor = V1beta1PredictorSpec(\n min_replicas=1,\n sklearn=V1beta1SKLearnSpec(\n storage_uri=\"gs://kfserving-examples/models/sklearn/1.0/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n sklearn_isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=sklearn_name,\n namespace=KSERVE_TEST_NAMESPACE,\n annotations=annotations,\n labels=labels,\n ),\n spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor),\n )\n\n xgb_predictor = V1beta1PredictorSpec(\n min_replicas=1,\n xgboost=V1beta1XGBoostSpec(\n storage_uri=\"gs://kfserving-examples/models/xgboost/1.5/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n xgb_isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=xgb_name,\n namespace=KSERVE_TEST_NAMESPACE,\n annotations=annotations,\n labels=labels,\n ),\n spec=V1beta1InferenceServiceSpec(predictor=xgb_predictor),\n )\n\n nodes = {\n \"root\": V1alpha1InferenceRouter(\n router_type=\"Sequence\",\n steps=[\n V1alpha1InferenceStep(\n service_name=sklearn_name,\n ),\n V1alpha1InferenceStep(\n service_name=xgb_name,\n data=\"$request\",\n ),\n ],\n )\n }\n graph_spec = V1alpha1InferenceGraphSpec(\n nodes=nodes,\n )\n ig = V1alpha1InferenceGraph(\n api_version=constants.KSERVE_V1ALPHA1,\n kind=constants.KSERVE_KIND_INFERENCEGRAPH,\n metadata=client.V1ObjectMeta(\n name=graph_name, namespace=KSERVE_TEST_NAMESPACE, annotations=annotations\n ),\n spec=graph_spec,\n )\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(sklearn_isvc)\n kserve_client.create(xgb_isvc)\n kserve_client.wait_isvc_ready(sklearn_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(xgb_name, namespace=KSERVE_TEST_NAMESPACE)\n\n kserve_client.create_inference_graph(ig)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n # Below checks are raw deployment specific. They ensure raw k8s resources created instead of knative resources\n dep = kserve_client.app_api.read_namespaced_deployment(\n graph_name, namespace=KSERVE_TEST_NAMESPACE\n )\n if not dep:\n raise RuntimeError(\n \"Deployment doesn't exist for InferenceGraph {} in raw deployment mode\".format(\n graph_name\n )\n )\n\n svc = kserve_client.core_api.read_namespaced_service(\n graph_name, namespace=KSERVE_TEST_NAMESPACE\n )\n if not svc:\n raise RuntimeError(\n \"Service doesn't exist for InferenceGraph {} in raw deployment mode\".format(\n graph_name\n )\n )\n\n try:\n knativeroute = kserve_client.api_instance.get_namespaced_custom_object(\n \"serving.knative.dev\", \"v1\", KSERVE_TEST_NAMESPACE, \"routes\", graph_name\n )\n if knativeroute:\n raise RuntimeError(\n \"Knative route resource shouldn't exist for InferenceGraph {}\".format(\n graph_name\n )\n + \"in raw deployment mode\"\n )\n except client.rest.ApiException:\n logger.info(\"Expected error in finding knative route in raw deployment mode\")\n\n try:\n knativesvc = kserve_client.api_instance.get_namespaced_custom_object(\n \"serving.knative.dev\", \"v1\", KSERVE_TEST_NAMESPACE, \"services\", graph_name\n )\n if knativesvc:\n raise RuntimeError(\n \"Knative resources shouldn't exist for InferenceGraph {} \".format(\n graph_name\n )\n + \"in raw deployment mode\"\n )\n except client.rest.ApiException:\n logger.info(\"Expected error in finding knative service in raw deployment mode\")\n\n # TODO Fix this when we enable ALB creation for IG raw deployment mode. This is required for traffic ingress\n # for this predict api call to work\n #\n # res = await predict_ig(\n # rest_v1_client,\n # graph_name,\n # os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, \"iris_input.json\"),\n # network_layer=network_layer,\n # )\n # assert res[\"predictions\"] == [1, 1]\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(sklearn_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(xgb_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.0001375439969706349, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "graph/test_inference_graph.py::test_inference_graph_raw_mode_with_hpa", "lineno": 1103, "outcome": "error", "keywords": ["test_inference_graph_raw_mode_with_hpa", "asyncio", "raw", "pytestmark", "test_inference_graph.py", "graph/__init__.py", "e2e"], "setup": {"duration": 0.0002371059963479638, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/graph/test_inference_graph.py, line 1104\n @pytest.mark.raw\n @pytest.mark.asyncio(scope=\"session\")\n async def test_inference_graph_raw_mode_with_hpa(rest_v1_client, network_layer):\n logger.info(\"Starting test test_inference_graph_raw_mode_with_hpa\")\n suffix = str(uuid.uuid4())[1:6]\n sklearn_name = \"isvc-sklearn-graph-raw-hpa-\" + suffix\n xgb_name = \"isvc-xgboost-graph-raw-hpa-\" + suffix\n graph_name = \"model-chainer-raw-hpa-\" + suffix\n\n annotations = dict()\n annotations[\"serving.kserve.io/deploymentMode\"] = \"Standard\"\n # annotations[\"serving.kserve.io/max-scale\"] = '5'\n # annotations[\"serving.kserve.io/metric\"] = 'rps'\n # annotations[\"serving.kserve.io/min-scale\"] = '2'\n # annotations[\"serving.kserve.io/target\"] = '30'\n labels = dict()\n labels[\"networking.kserve.io/visibility\"] = \"exposed\"\n\n sklearn_predictor = V1beta1PredictorSpec(\n min_replicas=1,\n sklearn=V1beta1SKLearnSpec(\n storage_uri=\"gs://kfserving-examples/models/sklearn/1.0/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n sklearn_isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=sklearn_name,\n namespace=KSERVE_TEST_NAMESPACE,\n annotations=annotations,\n labels=labels,\n ),\n spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor),\n )\n\n xgb_predictor = V1beta1PredictorSpec(\n min_replicas=1,\n xgboost=V1beta1XGBoostSpec(\n storage_uri=\"gs://kfserving-examples/models/xgboost/1.5/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"50m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n xgb_isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=xgb_name,\n namespace=KSERVE_TEST_NAMESPACE,\n annotations=annotations,\n labels=labels,\n ),\n spec=V1beta1InferenceServiceSpec(predictor=xgb_predictor),\n )\n\n nodes = {\n \"root\": V1alpha1InferenceRouter(\n router_type=\"Sequence\",\n steps=[\n V1alpha1InferenceStep(\n service_name=sklearn_name,\n ),\n V1alpha1InferenceStep(\n service_name=xgb_name,\n data=\"$request\",\n ),\n ],\n )\n }\n graph_spec = V1alpha1InferenceGraphSpec(\n nodes=nodes,\n )\n ig = V1alpha1InferenceGraph(\n api_version=constants.KSERVE_V1ALPHA1,\n kind=constants.KSERVE_KIND_INFERENCEGRAPH,\n metadata=client.V1ObjectMeta(\n name=graph_name, namespace=KSERVE_TEST_NAMESPACE, annotations=annotations\n ),\n spec=graph_spec,\n )\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(sklearn_isvc)\n kserve_client.create(xgb_isvc)\n kserve_client.wait_isvc_ready(sklearn_name, namespace=KSERVE_TEST_NAMESPACE)\n kserve_client.wait_isvc_ready(xgb_name, namespace=KSERVE_TEST_NAMESPACE)\n\n kserve_client.create_inference_graph(ig)\n kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)\n\n # Below checks are raw deployment specific. They ensure raw k8s resources created instead of knative resources\n dep = kserve_client.app_api.read_namespaced_deployment(\n graph_name, namespace=KSERVE_TEST_NAMESPACE\n )\n if not dep:\n raise RuntimeError(\n \"Deployment doesn't exist for InferenceGraph {} in raw deployment mode\".format(\n graph_name\n )\n )\n\n svc = kserve_client.core_api.read_namespaced_service(\n graph_name, namespace=KSERVE_TEST_NAMESPACE\n )\n if not svc:\n raise RuntimeError(\n \"Service doesn't exist for InferenceGraph {} in raw deployment mode\".format(\n graph_name\n )\n )\n\n # hpa = kserve_client.hpa_v2_api.read_namespaced_horizontal_pod_autoscaler(graph_name,\n # namespace=KSERVE_TEST_NAMESPACE)\n # if not hpa:\n # raise RuntimeError(\"HPA doesn't exist for InferenceGraph {} in raw deployment mode\".format(graph_name))\n\n try:\n knativeroute = kserve_client.api_instance.get_namespaced_custom_object(\n \"serving.knative.dev\", \"v1\", KSERVE_TEST_NAMESPACE, \"routes\", graph_name\n )\n if knativeroute:\n raise RuntimeError(\n \"Knative route resource shouldn't exist for InferenceGraph {} \".format(\n graph_name\n )\n + \"in raw deployment mode\"\n )\n except client.rest.ApiException:\n logger.info(\"Expected error in finding knative route in raw deployment mode\")\n\n try:\n knativesvc = kserve_client.api_instance.get_namespaced_custom_object(\n \"serving.knative.dev\", \"v1\", KSERVE_TEST_NAMESPACE, \"services\", graph_name\n )\n if knativesvc:\n raise RuntimeError(\n \"Knative resources shouldn't exist for InferenceGraph {} \".format(\n graph_name\n )\n + \"in raw deployment mode\"\n )\n except client.rest.ApiException:\n logger.info(\"Expected error in finding knative route in raw deployment mode\")\n\n # TODO Fix this when we enable ALB creation for IG raw deployment mode. This is required for traffic ingress\n # for this predict api call to work\n #\n # res = await predict_ig(\n # rest_v1_client,\n # graph_name,\n # os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, \"iris_input.json\"),\n # network_layer=network_layer,\n # )\n # assert res[\"predictions\"] == [1, 1]\n\n kserve_client.delete_inference_graph(graph_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(sklearn_name, KSERVE_TEST_NAMESPACE)\n kserve_client.delete(xgb_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, graph/__init__.py::, graph/test_inference_graph.py::, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.000149583000165876, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "logger/test_raw_logger.py::test_kserve_logger", "lineno": 39, "outcome": "error", "keywords": ["test_kserve_logger", "asyncio", "raw", "pytestmark", "test_raw_logger.py", "logger/__init__.py", "e2e"], "setup": {"duration": 0.0003924990014638752, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/logger/test_raw_logger.py, line 40\n @pytest.mark.raw\n @pytest.mark.asyncio(scope=\"session\")\n async def test_kserve_logger(rest_v1_client, network_layer):\n suffix = str(uuid.uuid4())[1:6]\n msg_dumper = \"message-dumper-raw-\" + suffix\n before(msg_dumper)\n\n service_name = \"isvc-logger-raw-\" + suffix\n predictor = V1beta1PredictorSpec(\n min_replicas=1,\n logger=V1beta1LoggerSpec(\n mode=\"all\",\n url=\"http://\"\n + msg_dumper\n + \"-predictor\"\n + \".\"\n + KSERVE_TEST_NAMESPACE\n + \".svc.cluster.local\",\n ),\n sklearn=V1beta1SKLearnSpec(\n storage_uri=\"gs://kfserving-examples/models/sklearn/1.0/model\",\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"10m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"100m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n await base_test(msg_dumper, service_name, predictor, rest_v1_client, network_layer)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, logger/test_raw_logger.py::, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00014314299914985895, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "predictor/test_autoscaling.py::test_sklearn_scale_raw", "lineno": 205, "outcome": "error", "keywords": ["test_sklearn_scale_raw", "asyncio", "raw", "pytestmark", "test_autoscaling.py", "predictor/__init__.py", "e2e"], "setup": {"duration": 0.00033593700209166855, "outcome": "failed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\nfile /workspace/source/test/e2e/predictor/test_autoscaling.py, line 206\n @pytest.mark.raw\n @pytest.mark.asyncio(scope=\"session\")\n async def test_sklearn_scale_raw(rest_v1_client, network_layer):\n suffix = str(uuid.uuid4())[1:6]\n service_name = \"isvc-sklearn-scale-raw-\" + suffix\n predictor = V1beta1PredictorSpec(\n min_replicas=1,\n scale_metric=\"cpu\",\n scale_target=50,\n sklearn=V1beta1SKLearnSpec(\n storage_uri=MODEL,\n resources=V1ResourceRequirements(\n requests={\"cpu\": \"25m\", \"memory\": \"128Mi\"},\n limits={\"cpu\": \"50m\", \"memory\": \"256Mi\"},\n ),\n ),\n )\n\n annotations = {\"serving.kserve.io/deploymentMode\": \"Standard\"}\n\n labels = dict()\n labels[\"networking.kserve.io/visibility\"] = \"exposed\"\n\n isvc = V1beta1InferenceService(\n api_version=constants.KSERVE_V1BETA1,\n kind=constants.KSERVE_KIND_INFERENCESERVICE,\n metadata=client.V1ObjectMeta(\n name=service_name,\n namespace=KSERVE_TEST_NAMESPACE,\n annotations=annotations,\n labels=labels,\n ),\n spec=V1beta1InferenceServiceSpec(predictor=predictor),\n )\n\n kserve_client = KServeClient(\n config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n )\n kserve_client.create(isvc)\n kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)\n api_instance = kserve_client.api_instance\n hpa_resp = api_instance.list_namespaced_custom_object(\n group=\"autoscaling\",\n version=\"v1\",\n namespace=KSERVE_TEST_NAMESPACE,\n label_selector=f\"serving.kserve.io/inferenceservice={service_name}\",\n plural=\"horizontalpodautoscalers\",\n )\n\n assert hpa_resp[\"items\"][0][\"spec\"][\"targetCPUUtilizationPercentage\"] == 50\n res = await predict_isvc(\n rest_v1_client, service_name, INPUT, network_layer=network_layer\n )\n assert res[\"predictions\"] == [1, 1]\n kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)\nfile /workspace/source/test/e2e/common/gateway_proxy_istio.py, line 121\n @pytest.fixture(autouse=True)\n def ensure_gateway_proxy_memory(test_case):\nE fixture 'test_case' not found\n> available fixtures: _cleanup_orphaned_predictor_isvcs, _session_event_loop, anyio_backend, anyio_backend_name, anyio_backend_options, assert_all_responses_were_requested, cache, capfd, capfdbinary, caplog, capsys, capsysbinary, configure_logger, cov, doctest_namespace, ensure_gateway_proxy_memory, event_loop, event_loop_policy, free_tcp_port, free_tcp_port_factory, free_udp_port, free_udp_port_factory, httpx_mock, include_metadata_in_junit_xml, json_metadata, metadata, monkeypatch, network_layer, no_cover, non_mocked_hosts, predictor/test_autoscaling.py::, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, rest_v1_client, rest_v2_client, testrun_uid, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory, unused_tcp_port, unused_tcp_port_factory, unused_udp_port, unused_udp_port_factory, worker_id\n> use 'pytest --fixtures [testpath]' for help on them.\n\n/workspace/source/test/e2e/common/gateway_proxy_istio.py:121"}, "teardown": {"duration": 0.00019990500004496425, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}]}