{"created": 1783003236.6353629, "duration": 864.8693301677704, "exitcode": 2, "root": "/workspace/source/test/e2e", "environment": {}, "summary": {"passed": 3, "skipped": 2, "failed": 5, "total": 10, "collected": 61}, "collectors": [{"nodeid": "explainer/test_art_explainer.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')"}, {"nodeid": "predictor/test_grpc.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')"}, {"nodeid": "predictor/test_torchserve.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')"}], "tests": [{"nodeid": "batcher/test_batcher.py::test_batcher", "lineno": 30, "outcome": "passed", "keywords": ["test_batcher", "asyncio", "predictor", "pytestmark", "test_batcher.py", "batcher/__init__.py", "e2e"], "setup": {"duration": 0.003807298999163322, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 115.38169167300111, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.00017942500016943086, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "batcher/test_batcher_custom_port.py::test_batcher_custom_port", "lineno": 32, "outcome": "passed", "keywords": ["test_batcher_custom_port", "asyncio", "predictor", "pytestmark", "test_batcher_custom_port.py", "batcher/__init__.py", "e2e"], "setup": {"duration": 0.00027787799990619533, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 95.2147734450009, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0002506680011720164, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "custom/test_custom_model_grpc.py::test_custom_model_grpc", "lineno": 41, "outcome": "skipped", "keywords": ["test_custom_model_grpc", "asyncio", "predictor", "grpc", "skip", "pytestmark", "test_custom_model_grpc.py", "custom/__init__.py", "e2e"], "setup": {"duration": 0.00016683499961800408, "outcome": "skipped", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n('/workspace/source/test/e2e/custom/test_custom_model_grpc.py', 42, 'Skipped: Not testable in ODH at the moment')"}, "teardown": {"duration": 0.00018944600014947355, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "custom/test_ray.py::test_custom_model_http_ray", "lineno": 29, "outcome": "skipped", "keywords": ["test_custom_model_http_ray", "asyncio", "predictor", "skip", "pytestmark", "test_ray.py", "custom/__init__.py", "e2e"], "setup": {"duration": 0.00014419499893847387, "outcome": "skipped", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n('/workspace/source/test/e2e/custom/test_ray.py', 30, 'Skipped: Not testable in ODH at the moment')"}, "teardown": {"duration": 0.00015124400124477688, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "logger/test_logger.py::test_kserve_logger", "lineno": 33, "outcome": "passed", "keywords": ["test_kserve_logger", "asyncio", "path_based_routing", "predictor", "pytestmark", "test_logger.py", "logger/__init__.py", "e2e"], "setup": {"duration": 0.0004897449998679804, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 115.34873359999983, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.00020252600006642751, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "predictor/test_autogluon.py::test_autogluon_runtime_kserve_v1", "lineno": 57, "outcome": "failed", "keywords": ["test_autogluon_runtime_kserve_v1", "asyncio", "predictor", "pytestmark", "test_autogluon.py", "predictor/__init__.py", "e2e"], "setup": {"duration": 0.0005975670010229805, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 170.27698132799924, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/common/utils.py", "lineno": 279, "message": "RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}"}, "traceback": [{"path": "predictor/test_autogluon.py", "lineno": 63, "message": ""}, {"path": "predictor/autogluon_helpers.py", "lineno": 96, "message": ""}, {"path": "common/utils.py", "lineno": 81, "message": ""}, {"path": "common/utils.py", "lineno": 372, "message": ""}, {"path": "common/utils.py", "lineno": 279, "message": "RuntimeError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nrest_v1_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\n\n    @pytest.mark.predictor\n    @pytest.mark.asyncio(scope=\"session\")\n    async def test_autogluon_runtime_kserve_v1(rest_v1_client):\n        service_name = \"isvc-autogluon-v1\"\n        predictor = _create_predictor(service_name)\n>       response = await deploy_and_predict(\n            service_name,\n            predictor,\n            rest_v1_client,\n            \"./data/autogluon_titanic_input.json\",\n        )\n\npredictor/test_autogluon.py:63: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nservice_name = 'isvc-autogluon-v1'\npredictor = {'active_deadline_seconds': None,\n 'affinity': None,\n 'annotations': None,\n 'auto_scaling': None,\n 'automount_service_... None,\n 'topology_spread_constraints': None,\n 'triton': None,\n 'volumes': None,\n 'worker_spec': None,\n 'xgboost': None}\nrest_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\ninput_path = './data/autogluon_titanic_input.json', timeout_seconds = 1200\n\n    async def deploy_and_predict(\n        service_name: str,\n        predictor: V1beta1PredictorSpec,\n        rest_client,\n        input_path: str,\n        timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT,\n    ):\n        async with autogluon_isvc(service_name, predictor, timeout_seconds):\n>           return await predict_isvc(rest_client, service_name, input_path)\n\npredictor/autogluon_helpers.py:96: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nclient = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\nservice_name = 'isvc-autogluon-v1'\ninput = './data/autogluon_titanic_input.json', version = 'v1beta1'\nmodel_name = None, is_batch = False, network_layer = 'istio'\nextra_headers = None\n\n    async def predict_isvc(\n        client: InferenceRESTClient,\n        service_name,\n        input: Union[str, InferRequest],\n        version=constants.KSERVE_V1BETA1_VERSION,\n        model_name=None,\n        is_batch=False,\n        network_layer: str = \"istio\",\n        extra_headers: dict = None,\n    ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:\n        kfs_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n        )\n        isvc = kfs_client.get(\n            service_name,\n            namespace=KSERVE_TEST_NAMESPACE,\n            version=version,\n        )\n>       scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer)\n\ncommon/utils.py:81: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nisvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...'1', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '1Gi'}}, 'runtime': 'kserve-autogluonserver', ...}}}, ...}\nnetwork_layer = 'istio'\n\n    def get_isvc_endpoint(isvc, network_layer: str = \"istio\"):\n        scheme = urlparse(isvc[\"status\"][\"url\"]).scheme\n        host = urlparse(isvc[\"status\"][\"url\"]).netloc\n        path = urlparse(isvc[\"status\"][\"url\"]).path\n        logger.info(f\"Host from isvc status URL = {host}\")\n        logger.info(f\"Network layer = {network_layer}\")\n        if network_layer == \"openshift-route\":\n            cluster_ip = host\n            logger.info(f\"Using external route host: {cluster_ip}\")\n        elif network_layer == \"istio\" or network_layer == \"istio-ingress\":\n>           cluster_ip = get_cluster_ip()\n\ncommon/utils.py:372: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nnamespace = 'istio-system'\nlabels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\n    def get_cluster_ip(namespace=\"istio-system\", labels: dict = None):\n        cluster_ip = os.environ.get(\"KSERVE_INGRESS_HOST_PORT\")\n        if cluster_ip is None:\n            api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient())\n            if labels is None:\n                labels = {\n                    \"app\": \"istio-ingressgateway\",\n                    \"istio\": \"ingressgateway\",\n                }\n            label_selector = \",\".join([f\"{key}={value}\" for key, value in labels.items()])\n            services = api_instance.list_namespaced_service(\n                namespace, label_selector=label_selector\n            )\n            if services.items:\n                service = services.items[0]\n            else:\n>               raise RuntimeError(f\"No service found with labels: {labels}\")\nE               RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\ncommon/utils.py:279: RuntimeError"}, "teardown": {"duration": 5.042451034998521, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "predictor/test_autogluon.py::test_autogluon_runtime_kserve_v2", "lineno": 72, "outcome": "failed", "keywords": ["test_autogluon_runtime_kserve_v2", "asyncio", "predictor", "pytestmark", "test_autogluon.py", "predictor/__init__.py", "e2e"], "setup": {"duration": 0.0013614690014946973, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 100.2346145320007, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/common/utils.py", "lineno": 279, "message": "RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}"}, "traceback": [{"path": "predictor/test_autogluon.py", "lineno": 78, "message": ""}, {"path": "predictor/autogluon_helpers.py", "lineno": 96, "message": ""}, {"path": "common/utils.py", "lineno": 81, "message": ""}, {"path": "common/utils.py", "lineno": 372, "message": ""}, {"path": "common/utils.py", "lineno": 279, "message": "RuntimeError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nrest_v2_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\n\n    @pytest.mark.predictor\n    @pytest.mark.asyncio(scope=\"session\")\n    async def test_autogluon_runtime_kserve_v2(rest_v2_client):\n        service_name = \"isvc-autogluon-v2\"\n        predictor = _create_predictor(service_name, protocol_version=\"v2\")\n>       response = await deploy_and_predict(\n            service_name,\n            predictor,\n            rest_v2_client,\n            \"./data/autogluon_titanic_input_v2.json\",\n        )\n\npredictor/test_autogluon.py:78: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nservice_name = 'isvc-autogluon-v2'\npredictor = {'active_deadline_seconds': None,\n 'affinity': None,\n 'annotations': None,\n 'auto_scaling': None,\n 'automount_service_... None,\n 'topology_spread_constraints': None,\n 'triton': None,\n 'volumes': None,\n 'worker_spec': None,\n 'xgboost': None}\nrest_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\ninput_path = './data/autogluon_titanic_input_v2.json', timeout_seconds = 1200\n\n    async def deploy_and_predict(\n        service_name: str,\n        predictor: V1beta1PredictorSpec,\n        rest_client,\n        input_path: str,\n        timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT,\n    ):\n        async with autogluon_isvc(service_name, predictor, timeout_seconds):\n>           return await predict_isvc(rest_client, service_name, input_path)\n\npredictor/autogluon_helpers.py:96: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nclient = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\nservice_name = 'isvc-autogluon-v2'\ninput = './data/autogluon_titanic_input_v2.json', version = 'v1beta1'\nmodel_name = None, is_batch = False, network_layer = 'istio'\nextra_headers = None\n\n    async def predict_isvc(\n        client: InferenceRESTClient,\n        service_name,\n        input: Union[str, InferRequest],\n        version=constants.KSERVE_V1BETA1_VERSION,\n        model_name=None,\n        is_batch=False,\n        network_layer: str = \"istio\",\n        extra_headers: dict = None,\n    ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:\n        kfs_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n        )\n        isvc = kfs_client.get(\n            service_name,\n            namespace=KSERVE_TEST_NAMESPACE,\n            version=version,\n        )\n>       scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer)\n\ncommon/utils.py:81: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nisvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...be': {'httpGet': {'path': '/v2/models/isvc-autogluon-v2/ready', 'port': 8080}, 'initialDelaySeconds': 90}, ...}}}, ...}\nnetwork_layer = 'istio'\n\n    def get_isvc_endpoint(isvc, network_layer: str = \"istio\"):\n        scheme = urlparse(isvc[\"status\"][\"url\"]).scheme\n        host = urlparse(isvc[\"status\"][\"url\"]).netloc\n        path = urlparse(isvc[\"status\"][\"url\"]).path\n        logger.info(f\"Host from isvc status URL = {host}\")\n        logger.info(f\"Network layer = {network_layer}\")\n        if network_layer == \"openshift-route\":\n            cluster_ip = host\n            logger.info(f\"Using external route host: {cluster_ip}\")\n        elif network_layer == \"istio\" or network_layer == \"istio-ingress\":\n>           cluster_ip = get_cluster_ip()\n\ncommon/utils.py:372: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nnamespace = 'istio-system'\nlabels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\n    def get_cluster_ip(namespace=\"istio-system\", labels: dict = None):\n        cluster_ip = os.environ.get(\"KSERVE_INGRESS_HOST_PORT\")\n        if cluster_ip is None:\n            api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient())\n            if labels is None:\n                labels = {\n                    \"app\": \"istio-ingressgateway\",\n                    \"istio\": \"ingressgateway\",\n                }\n            label_selector = \",\".join([f\"{key}={value}\" for key, value in labels.items()])\n            services = api_instance.list_namespaced_service(\n                namespace, label_selector=label_selector\n            )\n            if services.items:\n                service = services.items[0]\n            else:\n>               raise RuntimeError(f\"No service found with labels: {labels}\")\nE               RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\ncommon/utils.py:279: RuntimeError"}, "teardown": {"duration": 5.053352619999714, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "predictor/test_autogluon.py::test_autogluon_runtime_kserve_v2_input_variants", "lineno": 87, "outcome": "failed", "keywords": ["test_autogluon_runtime_kserve_v2_input_variants", "asyncio", "predictor", "pytestmark", "test_autogluon.py", "predictor/__init__.py", "e2e"], "setup": {"duration": 0.00043379300041124225, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 110.2283513729999, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/common/utils.py", "lineno": 279, "message": "RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}"}, "traceback": [{"path": "predictor/test_autogluon.py", "lineno": 99, "message": ""}, {"path": "common/utils.py", "lineno": 81, "message": ""}, {"path": "common/utils.py", "lineno": 372, "message": ""}, {"path": "common/utils.py", "lineno": 279, "message": "RuntimeError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nrest_v2_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\n\n    @pytest.mark.predictor\n    @pytest.mark.asyncio(scope=\"session\")\n    async def test_autogluon_runtime_kserve_v2_input_variants(rest_v2_client):\n        service_name = \"isvc-autogluon-v2-variants\"\n        predictor = _create_predictor(service_name, protocol_version=\"v2\")\n        async with autogluon_isvc(service_name, predictor):\n            for input_path in [\n                \"./data/autogluon_titanic_input_v2.json\",\n                \"./data/autogluon_titanic_input_v2_binary.json\",\n                \"./data/autogluon_titanic_input_v2_all_binary.json\",\n            ]:\n>               response = await predict_isvc(rest_v2_client, service_name, input_path)\n\npredictor/test_autogluon.py:99: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nclient = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\nservice_name = 'isvc-autogluon-v2-variants'\ninput = './data/autogluon_titanic_input_v2.json', version = 'v1beta1'\nmodel_name = None, is_batch = False, network_layer = 'istio'\nextra_headers = None\n\n    async def predict_isvc(\n        client: InferenceRESTClient,\n        service_name,\n        input: Union[str, InferRequest],\n        version=constants.KSERVE_V1BETA1_VERSION,\n        model_name=None,\n        is_batch=False,\n        network_layer: str = \"istio\",\n        extra_headers: dict = None,\n    ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:\n        kfs_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n        )\n        isvc = kfs_client.get(\n            service_name,\n            namespace=KSERVE_TEST_NAMESPACE,\n            version=version,\n        )\n>       scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer)\n\ncommon/utils.py:81: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nisvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...tpGet': {'path': '/v2/models/isvc-autogluon-v2-variants/ready', 'port': 8080}, 'initialDelaySeconds': 90}, ...}}}, ...}\nnetwork_layer = 'istio'\n\n    def get_isvc_endpoint(isvc, network_layer: str = \"istio\"):\n        scheme = urlparse(isvc[\"status\"][\"url\"]).scheme\n        host = urlparse(isvc[\"status\"][\"url\"]).netloc\n        path = urlparse(isvc[\"status\"][\"url\"]).path\n        logger.info(f\"Host from isvc status URL = {host}\")\n        logger.info(f\"Network layer = {network_layer}\")\n        if network_layer == \"openshift-route\":\n            cluster_ip = host\n            logger.info(f\"Using external route host: {cluster_ip}\")\n        elif network_layer == \"istio\" or network_layer == \"istio-ingress\":\n>           cluster_ip = get_cluster_ip()\n\ncommon/utils.py:372: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nnamespace = 'istio-system'\nlabels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\n    def get_cluster_ip(namespace=\"istio-system\", labels: dict = None):\n        cluster_ip = os.environ.get(\"KSERVE_INGRESS_HOST_PORT\")\n        if cluster_ip is None:\n            api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient())\n            if labels is None:\n                labels = {\n                    \"app\": \"istio-ingressgateway\",\n                    \"istio\": \"ingressgateway\",\n                }\n            label_selector = \",\".join([f\"{key}={value}\" for key, value in labels.items()])\n            services = api_instance.list_namespaced_service(\n                namespace, label_selector=label_selector\n            )\n            if services.items:\n                service = services.items[0]\n            else:\n>               raise RuntimeError(f\"No service found with labels: {labels}\")\nE               RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\ncommon/utils.py:279: RuntimeError"}, "teardown": {"duration": 5.057683969998834, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "predictor/test_autogluon.py::test_autogluon_runtime_kserve_v2_storage_uri_without_trailing_slash", "lineno": 103, "outcome": "failed", "keywords": ["test_autogluon_runtime_kserve_v2_storage_uri_without_trailing_slash", "asyncio", "predictor", "pytestmark", "test_autogluon.py", "predictor/__init__.py", "e2e"], "setup": {"duration": 0.0003903519991581561, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 100.20858563000002, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/common/utils.py", "lineno": 279, "message": "RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}"}, "traceback": [{"path": "predictor/test_autogluon.py", "lineno": 114, "message": ""}, {"path": "predictor/autogluon_helpers.py", "lineno": 96, "message": ""}, {"path": "common/utils.py", "lineno": 81, "message": ""}, {"path": "common/utils.py", "lineno": 372, "message": ""}, {"path": "common/utils.py", "lineno": 279, "message": "RuntimeError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nrest_v2_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\n\n    @pytest.mark.predictor\n    @pytest.mark.asyncio(scope=\"session\")\n    async def test_autogluon_runtime_kserve_v2_storage_uri_without_trailing_slash(\n        rest_v2_client,\n    ):\n        service_name = \"isvc-autogluon-v2-noslash\"\n        storage_uri = AUTOGLUON_STORAGE_URI.rstrip(\"/\")\n        predictor = _create_predictor(\n            service_name, protocol_version=\"v2\", storage_uri=storage_uri\n        )\n>       response = await deploy_and_predict(\n            service_name,\n            predictor,\n            rest_v2_client,\n            \"./data/autogluon_titanic_input_v2.json\",\n        )\n\npredictor/test_autogluon.py:114: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nservice_name = 'isvc-autogluon-v2-noslash'\npredictor = {'active_deadline_seconds': None,\n 'affinity': None,\n 'annotations': None,\n 'auto_scaling': None,\n 'automount_service_... None,\n 'topology_spread_constraints': None,\n 'triton': None,\n 'volumes': None,\n 'worker_spec': None,\n 'xgboost': None}\nrest_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\ninput_path = './data/autogluon_titanic_input_v2.json', timeout_seconds = 1200\n\n    async def deploy_and_predict(\n        service_name: str,\n        predictor: V1beta1PredictorSpec,\n        rest_client,\n        input_path: str,\n        timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT,\n    ):\n        async with autogluon_isvc(service_name, predictor, timeout_seconds):\n>           return await predict_isvc(rest_client, service_name, input_path)\n\npredictor/autogluon_helpers.py:96: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nclient = <kserve.inference_client.InferenceRESTClient object at 0x7f41f96b3710>\nservice_name = 'isvc-autogluon-v2-noslash'\ninput = './data/autogluon_titanic_input_v2.json', version = 'v1beta1'\nmodel_name = None, is_batch = False, network_layer = 'istio'\nextra_headers = None\n\n    async def predict_isvc(\n        client: InferenceRESTClient,\n        service_name,\n        input: Union[str, InferRequest],\n        version=constants.KSERVE_V1BETA1_VERSION,\n        model_name=None,\n        is_batch=False,\n        network_layer: str = \"istio\",\n        extra_headers: dict = None,\n    ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:\n        kfs_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n        )\n        isvc = kfs_client.get(\n            service_name,\n            namespace=KSERVE_TEST_NAMESPACE,\n            version=version,\n        )\n>       scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer)\n\ncommon/utils.py:81: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nisvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...ttpGet': {'path': '/v2/models/isvc-autogluon-v2-noslash/ready', 'port': 8080}, 'initialDelaySeconds': 90}, ...}}}, ...}\nnetwork_layer = 'istio'\n\n    def get_isvc_endpoint(isvc, network_layer: str = \"istio\"):\n        scheme = urlparse(isvc[\"status\"][\"url\"]).scheme\n        host = urlparse(isvc[\"status\"][\"url\"]).netloc\n        path = urlparse(isvc[\"status\"][\"url\"]).path\n        logger.info(f\"Host from isvc status URL = {host}\")\n        logger.info(f\"Network layer = {network_layer}\")\n        if network_layer == \"openshift-route\":\n            cluster_ip = host\n            logger.info(f\"Using external route host: {cluster_ip}\")\n        elif network_layer == \"istio\" or network_layer == \"istio-ingress\":\n>           cluster_ip = get_cluster_ip()\n\ncommon/utils.py:372: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nnamespace = 'istio-system'\nlabels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\n    def get_cluster_ip(namespace=\"istio-system\", labels: dict = None):\n        cluster_ip = os.environ.get(\"KSERVE_INGRESS_HOST_PORT\")\n        if cluster_ip is None:\n            api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient())\n            if labels is None:\n                labels = {\n                    \"app\": \"istio-ingressgateway\",\n                    \"istio\": \"ingressgateway\",\n                }\n            label_selector = \",\".join([f\"{key}={value}\" for key, value in labels.items()])\n            services = api_instance.list_namespaced_service(\n                namespace, label_selector=label_selector\n            )\n            if services.items:\n                service = services.items[0]\n            else:\n>               raise RuntimeError(f\"No service found with labels: {labels}\")\nE               RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\ncommon/utils.py:279: RuntimeError"}, "teardown": {"duration": 5.055236321999473, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "predictor/test_autogluon_timeseries.py::test_autogluon_timeseries_runtime_kserve_v1", "lineno": 68, "outcome": "failed", "keywords": ["test_autogluon_timeseries_runtime_kserve_v1", "asyncio", "predictor", "pytestmark", "test_autogluon_timeseries.py", "predictor/__init__.py", "e2e"], "setup": {"duration": 0.0004043909993924899, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 30.16952846299864, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/common/utils.py", "lineno": 279, "message": "RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}"}, "traceback": [{"path": "predictor/test_autogluon_timeseries.py", "lineno": 73, "message": ""}, {"path": "predictor/test_autogluon_timeseries.py", "lineno": 66, "message": ""}, {"path": "predictor/autogluon_helpers.py", "lineno": 96, "message": ""}, {"path": "common/utils.py", "lineno": 81, "message": ""}, {"path": "common/utils.py", "lineno": 372, "message": ""}, {"path": "common/utils.py", "lineno": 279, "message": "RuntimeError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nrest_v1_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\n\n    @pytest.mark.predictor\n    @pytest.mark.asyncio(scope=\"session\")\n    async def test_autogluon_timeseries_runtime_kserve_v1(rest_v1_client):\n        service_name = \"isvc-autogluon-ts-v1\"\n>       response = await _deploy_and_predict_v1(\n            service_name,\n            rest_v1_client,\n            \"./data/autogluon_timeseries_input.json\",\n        )\n\npredictor/test_autogluon_timeseries.py:73: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nservice_name = 'isvc-autogluon-ts-v1'\nrest_v1_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\ninput_path = './data/autogluon_timeseries_input.json', storage_uri = None\n\n    async def _deploy_and_predict_v1(\n        service_name: str, rest_v1_client, input_path: str, storage_uri: str = None\n    ):\n        predictor = _create_ts_predictor(service_name, storage_uri=storage_uri)\n>       return await deploy_and_predict(service_name, predictor, rest_v1_client, input_path)\n\npredictor/test_autogluon_timeseries.py:66: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nservice_name = 'isvc-autogluon-ts-v1'\npredictor = {'active_deadline_seconds': None,\n 'affinity': None,\n 'annotations': None,\n 'auto_scaling': None,\n 'automount_service_... None,\n 'topology_spread_constraints': None,\n 'triton': None,\n 'volumes': None,\n 'worker_spec': None,\n 'xgboost': None}\nrest_client = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\ninput_path = './data/autogluon_timeseries_input.json', timeout_seconds = 1200\n\n    async def deploy_and_predict(\n        service_name: str,\n        predictor: V1beta1PredictorSpec,\n        rest_client,\n        input_path: str,\n        timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT,\n    ):\n        async with autogluon_isvc(service_name, predictor, timeout_seconds):\n>           return await predict_isvc(rest_client, service_name, input_path)\n\npredictor/autogluon_helpers.py:96: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nclient = <kserve.inference_client.InferenceRESTClient object at 0x7f41f9c13a90>\nservice_name = 'isvc-autogluon-ts-v1'\ninput = './data/autogluon_timeseries_input.json', version = 'v1beta1'\nmodel_name = None, is_batch = False, network_layer = 'istio'\nextra_headers = None\n\n    async def predict_isvc(\n        client: InferenceRESTClient,\n        service_name,\n        input: Union[str, InferRequest],\n        version=constants.KSERVE_V1BETA1_VERSION,\n        model_name=None,\n        is_batch=False,\n        network_layer: str = \"istio\",\n        extra_headers: dict = None,\n    ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:\n        kfs_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\")\n        )\n        isvc = kfs_client.get(\n            service_name,\n            namespace=KSERVE_TEST_NAMESPACE,\n            version=version,\n        )\n>       scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer)\n\ncommon/utils.py:81: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nisvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...'2', 'memory': '4Gi'}, 'requests': {'cpu': '100m', 'memory': '2Gi'}}, 'runtime': 'kserve-autogluonserver', ...}}}, ...}\nnetwork_layer = 'istio'\n\n    def get_isvc_endpoint(isvc, network_layer: str = \"istio\"):\n        scheme = urlparse(isvc[\"status\"][\"url\"]).scheme\n        host = urlparse(isvc[\"status\"][\"url\"]).netloc\n        path = urlparse(isvc[\"status\"][\"url\"]).path\n        logger.info(f\"Host from isvc status URL = {host}\")\n        logger.info(f\"Network layer = {network_layer}\")\n        if network_layer == \"openshift-route\":\n            cluster_ip = host\n            logger.info(f\"Using external route host: {cluster_ip}\")\n        elif network_layer == \"istio\" or network_layer == \"istio-ingress\":\n>           cluster_ip = get_cluster_ip()\n\ncommon/utils.py:372: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nnamespace = 'istio-system'\nlabels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\n    def get_cluster_ip(namespace=\"istio-system\", labels: dict = None):\n        cluster_ip = os.environ.get(\"KSERVE_INGRESS_HOST_PORT\")\n        if cluster_ip is None:\n            api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient())\n            if labels is None:\n                labels = {\n                    \"app\": \"istio-ingressgateway\",\n                    \"istio\": \"ingressgateway\",\n                }\n            label_selector = \",\".join([f\"{key}={value}\" for key, value in labels.items()])\n            services = api_instance.list_namespaced_service(\n                namespace, label_selector=label_selector\n            )\n            if services.items:\n                service = services.items[0]\n            else:\n>               raise RuntimeError(f\"No service found with labels: {labels}\")\nE               RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'}\n\ncommon/utils.py:279: RuntimeError"}, "teardown": {"duration": 5.051351471000089, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}]}