('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')/workspace/source/test/e2e/custom/test_custom_model_grpc.py:42: Not testable in ODH at the moment/workspace/source/test/e2e/custom/test_ray.py:30: Not testable in ODH at the momentrest_v1_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_runtime_kserve_v1(rest_v1_client): service_name = "isvc-autogluon-v1" predictor = _create_predictor(service_name) > response = await deploy_and_predict( service_name, predictor, rest_v1_client, "./data/autogluon_titanic_input.json", ) predictor/test_autogluon.py:63: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ service_name = 'isvc-autogluon-v1' predictor = {'active_deadline_seconds': None, 'affinity': None, 'annotations': None, 'auto_scaling': None, 'automount_service_... None, 'topology_spread_constraints': None, 'triton': None, 'volumes': None, 'worker_spec': None, 'xgboost': None} rest_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> input_path = './data/autogluon_titanic_input.json', timeout_seconds = 1200 async def deploy_and_predict( service_name: str, predictor: V1beta1PredictorSpec, rest_client, input_path: str, timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT, ): async with autogluon_isvc(service_name, predictor, timeout_seconds): > return await predict_isvc(rest_client, service_name, input_path) predictor/autogluon_helpers.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> service_name = 'isvc-autogluon-v1' input = './data/autogluon_titanic_input.json', version = 'v1beta1' model_name = None, is_batch = False, network_layer = 'istio' extra_headers = None async def predict_isvc( client: InferenceRESTClient, service_name, input: Union[str, InferRequest], version=constants.KSERVE_V1BETA1_VERSION, model_name=None, is_batch=False, network_layer: str = "istio", extra_headers: dict = None, ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]: kfs_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) isvc = kfs_client.get( service_name, namespace=KSERVE_TEST_NAMESPACE, version=version, ) > scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer) common/utils.py:81: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ isvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...'1', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '1Gi'}}, 'runtime': 'kserve-autogluonserver', ...}}}, ...} network_layer = 'istio' def get_isvc_endpoint(isvc, network_layer: str = "istio"): scheme = urlparse(isvc["status"]["url"]).scheme host = urlparse(isvc["status"]["url"]).netloc path = urlparse(isvc["status"]["url"]).path logger.info(f"Host from isvc status URL = {host}") logger.info(f"Network layer = {network_layer}") if network_layer == "openshift-route": cluster_ip = host logger.info(f"Using external route host: {cluster_ip}") elif network_layer == "istio" or network_layer == "istio-ingress": > cluster_ip = get_cluster_ip() common/utils.py:372: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ namespace = 'istio-system' labels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} def get_cluster_ip(namespace="istio-system", labels: dict = None): cluster_ip = os.environ.get("KSERVE_INGRESS_HOST_PORT") if cluster_ip is None: api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient()) if labels is None: labels = { "app": "istio-ingressgateway", "istio": "ingressgateway", } label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) services = api_instance.list_namespaced_service( namespace, label_selector=label_selector ) if services.items: service = services.items[0] else: > raise RuntimeError(f"No service found with labels: {labels}") E RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} common/utils.py:279: RuntimeErrorrest_v2_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_runtime_kserve_v2(rest_v2_client): service_name = "isvc-autogluon-v2" predictor = _create_predictor(service_name, protocol_version="v2") > response = await deploy_and_predict( service_name, predictor, rest_v2_client, "./data/autogluon_titanic_input_v2.json", ) predictor/test_autogluon.py:78: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ service_name = 'isvc-autogluon-v2' predictor = {'active_deadline_seconds': None, 'affinity': None, 'annotations': None, 'auto_scaling': None, 'automount_service_... None, 'topology_spread_constraints': None, 'triton': None, 'volumes': None, 'worker_spec': None, 'xgboost': None} rest_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> input_path = './data/autogluon_titanic_input_v2.json', timeout_seconds = 1200 async def deploy_and_predict( service_name: str, predictor: V1beta1PredictorSpec, rest_client, input_path: str, timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT, ): async with autogluon_isvc(service_name, predictor, timeout_seconds): > return await predict_isvc(rest_client, service_name, input_path) predictor/autogluon_helpers.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> service_name = 'isvc-autogluon-v2' input = './data/autogluon_titanic_input_v2.json', version = 'v1beta1' model_name = None, is_batch = False, network_layer = 'istio' extra_headers = None async def predict_isvc( client: InferenceRESTClient, service_name, input: Union[str, InferRequest], version=constants.KSERVE_V1BETA1_VERSION, model_name=None, is_batch=False, network_layer: str = "istio", extra_headers: dict = None, ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]: kfs_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) isvc = kfs_client.get( service_name, namespace=KSERVE_TEST_NAMESPACE, version=version, ) > scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer) common/utils.py:81: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ isvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...be': {'httpGet': {'path': '/v2/models/isvc-autogluon-v2/ready', 'port': 8080}, 'initialDelaySeconds': 90}, ...}}}, ...} network_layer = 'istio' def get_isvc_endpoint(isvc, network_layer: str = "istio"): scheme = urlparse(isvc["status"]["url"]).scheme host = urlparse(isvc["status"]["url"]).netloc path = urlparse(isvc["status"]["url"]).path logger.info(f"Host from isvc status URL = {host}") logger.info(f"Network layer = {network_layer}") if network_layer == "openshift-route": cluster_ip = host logger.info(f"Using external route host: {cluster_ip}") elif network_layer == "istio" or network_layer == "istio-ingress": > cluster_ip = get_cluster_ip() common/utils.py:372: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ namespace = 'istio-system' labels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} def get_cluster_ip(namespace="istio-system", labels: dict = None): cluster_ip = os.environ.get("KSERVE_INGRESS_HOST_PORT") if cluster_ip is None: api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient()) if labels is None: labels = { "app": "istio-ingressgateway", "istio": "ingressgateway", } label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) services = api_instance.list_namespaced_service( namespace, label_selector=label_selector ) if services.items: service = services.items[0] else: > raise RuntimeError(f"No service found with labels: {labels}") E RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} common/utils.py:279: RuntimeErrorrest_v2_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_runtime_kserve_v2_input_variants(rest_v2_client): service_name = "isvc-autogluon-v2-variants" predictor = _create_predictor(service_name, protocol_version="v2") async with autogluon_isvc(service_name, predictor): for input_path in [ "./data/autogluon_titanic_input_v2.json", "./data/autogluon_titanic_input_v2_binary.json", "./data/autogluon_titanic_input_v2_all_binary.json", ]: > response = await predict_isvc(rest_v2_client, service_name, input_path) predictor/test_autogluon.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> service_name = 'isvc-autogluon-v2-variants' input = './data/autogluon_titanic_input_v2.json', version = 'v1beta1' model_name = None, is_batch = False, network_layer = 'istio' extra_headers = None async def predict_isvc( client: InferenceRESTClient, service_name, input: Union[str, InferRequest], version=constants.KSERVE_V1BETA1_VERSION, model_name=None, is_batch=False, network_layer: str = "istio", extra_headers: dict = None, ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]: kfs_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) isvc = kfs_client.get( service_name, namespace=KSERVE_TEST_NAMESPACE, version=version, ) > scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer) common/utils.py:81: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ isvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...tpGet': {'path': '/v2/models/isvc-autogluon-v2-variants/ready', 'port': 8080}, 'initialDelaySeconds': 90}, ...}}}, ...} network_layer = 'istio' def get_isvc_endpoint(isvc, network_layer: str = "istio"): scheme = urlparse(isvc["status"]["url"]).scheme host = urlparse(isvc["status"]["url"]).netloc path = urlparse(isvc["status"]["url"]).path logger.info(f"Host from isvc status URL = {host}") logger.info(f"Network layer = {network_layer}") if network_layer == "openshift-route": cluster_ip = host logger.info(f"Using external route host: {cluster_ip}") elif network_layer == "istio" or network_layer == "istio-ingress": > cluster_ip = get_cluster_ip() common/utils.py:372: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ namespace = 'istio-system' labels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} def get_cluster_ip(namespace="istio-system", labels: dict = None): cluster_ip = os.environ.get("KSERVE_INGRESS_HOST_PORT") if cluster_ip is None: api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient()) if labels is None: labels = { "app": "istio-ingressgateway", "istio": "ingressgateway", } label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) services = api_instance.list_namespaced_service( namespace, label_selector=label_selector ) if services.items: service = services.items[0] else: > raise RuntimeError(f"No service found with labels: {labels}") E RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} common/utils.py:279: RuntimeErrorrest_v2_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_runtime_kserve_v2_storage_uri_without_trailing_slash( rest_v2_client, ): service_name = "isvc-autogluon-v2-noslash" storage_uri = AUTOGLUON_STORAGE_URI.rstrip("/") predictor = _create_predictor( service_name, protocol_version="v2", storage_uri=storage_uri ) > response = await deploy_and_predict( service_name, predictor, rest_v2_client, "./data/autogluon_titanic_input_v2.json", ) predictor/test_autogluon.py:114: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ service_name = 'isvc-autogluon-v2-noslash' predictor = {'active_deadline_seconds': None, 'affinity': None, 'annotations': None, 'auto_scaling': None, 'automount_service_... None, 'topology_spread_constraints': None, 'triton': None, 'volumes': None, 'worker_spec': None, 'xgboost': None} rest_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> input_path = './data/autogluon_titanic_input_v2.json', timeout_seconds = 1200 async def deploy_and_predict( service_name: str, predictor: V1beta1PredictorSpec, rest_client, input_path: str, timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT, ): async with autogluon_isvc(service_name, predictor, timeout_seconds): > return await predict_isvc(rest_client, service_name, input_path) predictor/autogluon_helpers.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1cf6d050> service_name = 'isvc-autogluon-v2-noslash' input = './data/autogluon_titanic_input_v2.json', version = 'v1beta1' model_name = None, is_batch = False, network_layer = 'istio' extra_headers = None async def predict_isvc( client: InferenceRESTClient, service_name, input: Union[str, InferRequest], version=constants.KSERVE_V1BETA1_VERSION, model_name=None, is_batch=False, network_layer: str = "istio", extra_headers: dict = None, ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]: kfs_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) isvc = kfs_client.get( service_name, namespace=KSERVE_TEST_NAMESPACE, version=version, ) > scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer) common/utils.py:81: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ isvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...ttpGet': {'path': '/v2/models/isvc-autogluon-v2-noslash/ready', 'port': 8080}, 'initialDelaySeconds': 90}, ...}}}, ...} network_layer = 'istio' def get_isvc_endpoint(isvc, network_layer: str = "istio"): scheme = urlparse(isvc["status"]["url"]).scheme host = urlparse(isvc["status"]["url"]).netloc path = urlparse(isvc["status"]["url"]).path logger.info(f"Host from isvc status URL = {host}") logger.info(f"Network layer = {network_layer}") if network_layer == "openshift-route": cluster_ip = host logger.info(f"Using external route host: {cluster_ip}") elif network_layer == "istio" or network_layer == "istio-ingress": > cluster_ip = get_cluster_ip() common/utils.py:372: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ namespace = 'istio-system' labels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} def get_cluster_ip(namespace="istio-system", labels: dict = None): cluster_ip = os.environ.get("KSERVE_INGRESS_HOST_PORT") if cluster_ip is None: api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient()) if labels is None: labels = { "app": "istio-ingressgateway", "istio": "ingressgateway", } label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) services = api_instance.list_namespaced_service( namespace, label_selector=label_selector ) if services.items: service = services.items[0] else: > raise RuntimeError(f"No service found with labels: {labels}") E RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} common/utils.py:279: RuntimeErrorrest_v1_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> @pytest.mark.predictor @pytest.mark.asyncio(scope="session") async def test_autogluon_timeseries_runtime_kserve_v1(rest_v1_client): service_name = "isvc-autogluon-ts-v1" > response = await _deploy_and_predict_v1( service_name, rest_v1_client, "./data/autogluon_timeseries_input.json", ) predictor/test_autogluon_timeseries.py:73: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ service_name = 'isvc-autogluon-ts-v1' rest_v1_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> input_path = './data/autogluon_timeseries_input.json', storage_uri = None async def _deploy_and_predict_v1( service_name: str, rest_v1_client, input_path: str, storage_uri: str = None ): predictor = _create_ts_predictor(service_name, storage_uri=storage_uri) > return await deploy_and_predict(service_name, predictor, rest_v1_client, input_path) predictor/test_autogluon_timeseries.py:66: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ service_name = 'isvc-autogluon-ts-v1' predictor = {'active_deadline_seconds': None, 'affinity': None, 'annotations': None, 'auto_scaling': None, 'automount_service_... None, 'topology_spread_constraints': None, 'triton': None, 'volumes': None, 'worker_spec': None, 'xgboost': None} rest_client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> input_path = './data/autogluon_timeseries_input.json', timeout_seconds = 1200 async def deploy_and_predict( service_name: str, predictor: V1beta1PredictorSpec, rest_client, input_path: str, timeout_seconds: int = AUTOGLUON_ISVC_WAIT_TIMEOUT, ): async with autogluon_isvc(service_name, predictor, timeout_seconds): > return await predict_isvc(rest_client, service_name, input_path) predictor/autogluon_helpers.py:96: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ client = <kserve.inference_client.InferenceRESTClient object at 0x7fdc1bc0ff10> service_name = 'isvc-autogluon-ts-v1' input = './data/autogluon_timeseries_input.json', version = 'v1beta1' model_name = None, is_batch = False, network_layer = 'istio' extra_headers = None async def predict_isvc( client: InferenceRESTClient, service_name, input: Union[str, InferRequest], version=constants.KSERVE_V1BETA1_VERSION, model_name=None, is_batch=False, network_layer: str = "istio", extra_headers: dict = None, ) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]: kfs_client = KServeClient( config_file=os.environ.get("KUBECONFIG", "~/.kube/config") ) isvc = kfs_client.get( service_name, namespace=KSERVE_TEST_NAMESPACE, version=version, ) > scheme, cluster_ip, host, path = get_isvc_endpoint(isvc, network_layer) common/utils.py:81: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ isvc = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'annotations': {'modelFormat': 'a...'2', 'memory': '4Gi'}, 'requests': {'cpu': '100m', 'memory': '2Gi'}}, 'runtime': 'kserve-autogluonserver', ...}}}, ...} network_layer = 'istio' def get_isvc_endpoint(isvc, network_layer: str = "istio"): scheme = urlparse(isvc["status"]["url"]).scheme host = urlparse(isvc["status"]["url"]).netloc path = urlparse(isvc["status"]["url"]).path logger.info(f"Host from isvc status URL = {host}") logger.info(f"Network layer = {network_layer}") if network_layer == "openshift-route": cluster_ip = host logger.info(f"Using external route host: {cluster_ip}") elif network_layer == "istio" or network_layer == "istio-ingress": > cluster_ip = get_cluster_ip() common/utils.py:372: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ namespace = 'istio-system' labels = {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} def get_cluster_ip(namespace="istio-system", labels: dict = None): cluster_ip = os.environ.get("KSERVE_INGRESS_HOST_PORT") if cluster_ip is None: api_instance = k8s_client.CoreV1Api(k8s_client.ApiClient()) if labels is None: labels = { "app": "istio-ingressgateway", "istio": "ingressgateway", } label_selector = ",".join([f"{key}={value}" for key, value in labels.items()]) services = api_instance.list_namespaced_service( namespace, label_selector=label_selector ) if services.items: service = services.items[0] else: > raise RuntimeError(f"No service found with labels: {labels}") E RuntimeError: No service found with labels: {'app': 'istio-ingressgateway', 'istio': 'ingressgateway'} common/utils.py:279: RuntimeError