Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/stream/ws_client.py", line 533, in websocket_call client = WSClient(configuration, url, headers, capture_all, binary=binary) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/stream/ws_client.py", line 68, in __init__ self.sock = create_websocket(configuration, url, headers) ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/stream/ws_client.py", line 499, in create_websocket websocket.connect(url, **connect_opt) ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/websocket/_core.py", line 263, in connect self.sock, addrs = connect( ~~~~~~~^ url, self.sock_opt, proxy_info(**options), options.pop("socket", None) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/websocket/_http.py", line 146, in connect sock = _open_socket(addrinfo_list, options.sockopt, options.timeout) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/websocket/_http.py", line 223, in _open_socket raise error File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/websocket/_http.py", line 210, in _open_socket sock.connect(address) ~~~~~~~~~~~~^^^^^^^^^ TimeoutError: [Errno 110] Connection timed out During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 900, in pytest_runtest_call return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/skipping.py", line 268, in pytest_runtest_call return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 184, in pytest_runtest_call item.runtest() ~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1707, in runtest self.ihook.pytest_pyfunc_call(pyfuncitem=self) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 167, in pytest_pyfunc_call result = testfunction(**testargs) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/mlserver/probes/test_mlserver_probes.py", line 59, in test_mlserver_readiness_probe status_code = exec_mlserver_health_check( pod=mlserver_probes_pod_resource, http_get=resolve_http_get(probe=readiness_probe) ) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/mlserver/probes/utils.py", line 127, in exec_mlserver_health_check return exec_http_probe(pod=pod, http_get=http_get) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/mlserver/probes/utils.py", line 122, in exec_http_probe return pod.execute(command=curl_cmd, container=Containers.KSERVE_CONTAINER_NAME).strip() ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/pod.py", line 679, in execute resp = kubernetes.stream.stream( api_method=self._kube_v1_api.connect_get_namespaced_pod_exec, ...<8 lines>... _preload_content=False, ) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/stream/stream.py", line 36, in _websocket_request out = api_method(*args, **kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/client/api/core_v1_api.py", line 994, in connect_get_namespaced_pod_exec return self.connect_get_namespaced_pod_exec_with_http_info(name, namespace, **kwargs) # noqa: E501 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/client/api/core_v1_api.py", line 1101, in connect_get_namespaced_pod_exec_with_http_info return self.api_client.call_api( ~~~~~~~~~~~~~~~~~~~~~~~~^ '/api/v1/namespaces/{namespace}/pods/{name}/exec', 'GET', ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<11 lines>... _request_timeout=local_var_params.get('_request_timeout'), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ collection_formats=collection_formats) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/client/api_client.py", line 348, in call_api return self.__call_api(resource_path, method, ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^ path_params, query_params, header_params, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<2 lines>... _return_http_data_only, collection_formats, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _preload_content, _request_timeout, _host) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/client/api_client.py", line 180, in __call_api response_data = self.request( method, url, query_params=query_params, headers=header_params, post_params=post_params, body=body, _preload_content=_preload_content, _request_timeout=_request_timeout) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/kubernetes/stream/ws_client.py", line 543, in websocket_call raise ApiException(status=0, reason=str(e)) kubernetes.client.exceptions.ApiException: (0) Reason: [Errno 110] Connection timed outTraceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 900, in pytest_runtest_call return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/skipping.py", line 268, in pytest_runtest_call return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 184, in pytest_runtest_call item.runtest() ~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1707, in runtest self.ihook.pytest_pyfunc_call(pyfuncitem=self) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 167, in pytest_pyfunc_call result = testfunction(**testargs) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/pvc/test_triton_pvc_inference.py", line 86, in test_triton_pvc_inference validate_inference_request( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ pod_name=triton_pod_resource.name, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<5 lines>... root_dir=root_dir, ^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/S3/utils.py", line 139, in validate_inference_request response = run_triton_inference( pod_name=pod_name, ...<4 lines>... root_dir=root_dir, ) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/S3/utils.py", line 115, in run_triton_inference with portforward.forward(pod_or_service=pod_name, namespace=isvc.namespace, from_port=port, to_port=port): ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/portforward/__init__.py", line 89, in forward raise PortforwardError(err) from None portforward.PortforwardError: ServiceError: client error (Connect) Caused by: 0: client error (Connect) 1: deadline has elapsed