{"created": 1783431889.8010852, "duration": 6748.198301315308, "exitcode": 2, "root": "/workspace/source/test/e2e", "environment": {}, "summary": {"passed": 33, "failed": 7, "total": 40, "collected": 42}, "collectors": [{"nodeid": "explainer/test_art_explainer.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/explainer/test_art_explainer.py', 38, 'Skipped: ODH does not support art explainer at the moment')"}, {"nodeid": "predictor/test_grpc.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_grpc.py', 35, 'Skipped: Not testable in ODH at the moment')"}, {"nodeid": "predictor/test_torchserve.py", "outcome": "skipped", "result": [], "longrepr": "('/workspace/source/test/e2e/predictor/test_torchserve.py', 34, 'Skipped: ODH does not support torchserve at the moment')"}], "tests": [{"nodeid": "llmisvc/test_gateway_section_name.py::test_gateway_section_name_propagation[cluster_single_node-cluster_cpu-with-section-name]", "lineno": 131, "outcome": "passed", "keywords": ["test_gateway_section_name_propagation[with-section-name]", "parametrize", "llmd_simulator", "cluster_single_node", "cluster_cpu", "llminferenceservice", "pytestmark", "with-section-name", "test_gateway_section_name.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.0016554399990127422, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 57.4476136580015, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0005803669992019422, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.32813141899896436, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 111.83984879000127, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0046833110027364455, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_gateway_section_name.py::test_gateway_section_name_propagation[cluster_single_node-cluster_cpu-without-section-name]", "lineno": 131, "outcome": "passed", "keywords": ["test_gateway_section_name_propagation[without-section-name]", "parametrize", "llmd_simulator", "cluster_single_node", "cluster_cpu", "llminferenceservice", "pytestmark", "without-section-name", "test_gateway_section_name.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.0003448509996815119, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 26.9197183230026, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0005597069975920022, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_auth.py::test_llm_auth_enabled_requires_token[cluster_cpu-cluster_single_node-auth-enabled-default]", "lineno": 221, "outcome": "passed", "keywords": ["test_llm_auth_enabled_requires_token[auth-enabled-default]", "parametrize", "auth", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "auth-enabled-default", "test_llm_auth.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.1759715549997054, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 181.04842757699953, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0023687899993092287, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator0]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator0]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator0", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.2543340409974917, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 86.38443960100267, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0018248449996463023, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator1]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator1]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "model_routing", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator1", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.1896580049979093, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 157.00881198299976, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.006456483002693858, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_auth.py::test_llm_auth_invalid_token_rejected[cluster_cpu-cluster_single_node-auth-invalid-token]", "lineno": 380, "outcome": "passed", "keywords": ["test_llm_auth_invalid_token_rejected[auth-invalid-token]", "parametrize", "auth", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "auth-invalid-token", "test_llm_auth.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.34000882699911017, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 187.6966176370006, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0027735529984056484, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator2]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator2]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "model_routing", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator2", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.19135373100289144, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 200.7950741429995, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004501004001213005, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_auth.py::test_llm_auth_disabled_no_token_required[cluster_cpu-cluster_single_node-auth-disabled]", "lineno": 511, "outcome": "passed", "keywords": ["test_llm_auth_disabled_no_token_required[auth-disabled]", "parametrize", "auth", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "auth-disabled", "test_llm_auth.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.13379228999838233, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 157.41449417400145, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.002271817000291776, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0]", "lineno": 244, "outcome": "failed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "model_routing", "lora", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.14182725900172954, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 1006.8705897279979, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))"}, "traceback": [{"path": "llmisvc/test_llm_inference_service.py", "lineno": 816, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1119, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError"}], "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebc1155d0>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebc1155d0>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n>           response = conn.getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:534: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebc1155d0>\n\n    def getresponse(  # type: ignore[override]\n        self,\n    ) -> HTTPResponse:\n        \"\"\"\n        Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.\n    \n        If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.\n        \"\"\"\n        # Raise the same error as http.client.HTTPConnection\n        if self._response_options is None:\n            raise ResponseNotReady()\n    \n        # Reset this attribute for being used again.\n        resp_options = self._response_options\n        self._response_options = None\n    \n        # Since the connection's timeout value may have been updated\n        # we need to set the timeout on the socket.\n        self.sock.settimeout(self.timeout)\n    \n        # This is needed here to avoid circular import errors\n        from .response import HTTPResponse\n    \n        # Save a reference to the shutdown function before ownership is passed\n        # to httplib_response\n        # TODO should we implement it everywhere?\n        _shutdown = getattr(self.sock, \"shutdown\", None)\n    \n        # Get the response from http.client.HTTPConnection\n>       httplib_response = super().getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:571: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebc1155d0>\n\n    def getresponse(self):\n        \"\"\"Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an\n        instance of HTTPResponse or of whatever object is returned by\n        the response_class variable.\n    \n        If a request has not been sent or if a previous response has\n        not be handled, ResponseNotReady is raised.  If the HTTP\n        response indicates that the connection should be closed, then\n        it will be closed before the response is returned.  When the\n        connection is closed, the underlying socket is closed.\n        \"\"\"\n    \n        # if a prior response has been completed, then forget about it.\n        if self.__response and self.__response.isclosed():\n            self.__response = None\n    \n        # if a prior response exists, then it must be completed (otherwise, we\n        # cannot read this response's header to determine the connection-close\n        # behavior)\n        #\n        # note: if a prior response existed, but was connection-close, then the\n        # socket and response were made independent of this HTTPConnection\n        # object since a new request requires that we open a whole new\n        # connection\n        #\n        # this means the prior response had one of two states:\n        #   1) will_close: this connection was reset and the prior socket and\n        #                  response operate independently\n        #   2) persistent: the response was retained and we await its\n        #                  isclosed() status to become true.\n        #\n        if self.__state != _CS_REQ_SENT or self.__response:\n            raise ResponseNotReady(self.__state)\n    \n        if self.debuglevel > 0:\n            response = self.response_class(self.sock, self.debuglevel,\n                                           method=self._method)\n        else:\n            response = self.response_class(self.sock, method=self._method)\n    \n        try:\n            try:\n>               response.begin()\n\n/usr/lib64/python3.11/http/client.py:1395: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7fcebc44b190>\n\n    def begin(self):\n        if self.headers is not None:\n            # we've already started reading the response\n            return\n    \n        # read until we get a non-100 response\n        while True:\n>           version, status, reason = self._read_status()\n\n/usr/lib64/python3.11/http/client.py:325: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7fcebc44b190>\n\n    def _read_status(self):\n>       line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n\n/usr/lib64/python3.11/http/client.py:286: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <socket.SocketIO object at 0x7fcebc44ace0>\nb = <memory at 0x7fcebc0c6680>\n\n    def readinto(self, b):\n        \"\"\"Read up to len(b) bytes into the writable buffer *b* and return\n        the number of bytes read.  If the socket is non-blocking and no bytes\n        are available, None is returned.\n    \n        If *b* is non-empty, a 0 return value indicates that the connection\n        was shutdown at the other end.\n        \"\"\"\n        self._checkClosed()\n        self._checkReadable()\n        if self._timeout_occurred:\n            raise OSError(\"cannot read from timed out object\")\n        while True:\n            try:\n>               return self._sock.recv_into(b)\nE               TimeoutError: timed out\n\n/usr/lib64/python3.11/socket.py:718: TimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n>           response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:787: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebc1155d0>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebc1155d0>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n            response = conn.getresponse()\n        except (BaseSSLError, OSError) as e:\n>           self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:536: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nerr = TimeoutError('timed out'), url = '/v1/completions', timeout_value = 60\n\n    def _raise_timeout(\n        self,\n        err: BaseSSLError | OSError | SocketTimeout,\n        url: str,\n        timeout_value: _TYPE_TIMEOUT | None,\n    ) -> None:\n        \"\"\"Is the error actually a timeout? Will raise a ReadTimeout or pass\"\"\"\n    \n        if isinstance(err, SocketTimeout):\n>           raise ReadTimeoutError(\n                self, url, f\"Read timed out. (read timeout={timeout_value})\"\n            ) from err\nE           urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <requests.adapters.HTTPAdapter object at 0x7fcebc06f350>\nrequest = <PreparedRequest [POST]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n>           resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:667: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=7, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=6, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=5, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=4, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=3, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=2, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=1, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\nmethod = 'POST', url = '/v1/completions'\nbody = b'{\"model\": \"publishers/kserve-ci-e2e-test/models/lora-adapter-1\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...lication/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'Content-Length': '107'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n>           retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:841: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nmethod = 'POST', url = '/v1/completions', response = None\nerror = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\n_pool = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebc06e410>\n_stacktrace = <traceback object at 0x7fcebc117e00>\n\n    def increment(\n        self,\n        method: str | None = None,\n        url: str | None = None,\n        response: BaseHTTPResponse | None = None,\n        error: Exception | None = None,\n        _pool: ConnectionPool | None = None,\n        _stacktrace: TracebackType | None = None,\n    ) -> Self:\n        \"\"\"Return a new Retry object with incremented retry counters.\n    \n        :param response: A response object, or None, if the server did not\n            return a response.\n        :type response: :class:`~urllib3.response.BaseHTTPResponse`\n        :param Exception error: An error encountered during the request, or\n            None if the response was received successfully.\n    \n        :return: A new ``Retry`` object.\n        \"\"\"\n        if self.total is False and error:\n            # Disabled, indicate to re-raise the error.\n            raise reraise(type(error), error, _stacktrace)\n    \n        total = self.total\n        if total is not None:\n            total -= 1\n    \n        connect = self.connect\n        read = self.read\n        redirect = self.redirect\n        status_count = self.status\n        other = self.other\n        cause = \"unknown\"\n        status = None\n        redirect_location = None\n    \n        if error and self._is_connection_error(error):\n            # Connect retry?\n            if connect is False:\n                raise reraise(type(error), error, _stacktrace)\n            elif connect is not None:\n                connect -= 1\n    \n        elif error and self._is_read_error(error):\n            # Read retry?\n            if read is False or method is None or not self._is_method_retryable(method):\n                raise reraise(type(error), error, _stacktrace)\n            elif read is not None:\n                read -= 1\n    \n        elif error:\n            # Other retry?\n            if other is not None:\n                other -= 1\n    \n        elif response and response.get_redirect_location():\n            # Redirect retry?\n            if redirect is not None:\n                redirect -= 1\n            cause = \"too many redirects\"\n            response_redirect_location = response.get_redirect_location()\n            if response_redirect_location:\n                redirect_location = response_redirect_location\n            status = response.status\n    \n        else:\n            # Incrementing because of a server error like a 500 in\n            # status_forcelist and the given method is in the allowed_methods\n            cause = ResponseError.GENERIC_ERROR\n            if response and response.status:\n                if status_count is not None:\n                    status_count -= 1\n                cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)\n                status = response.status\n    \n        history = self.history + (\n            RequestHistory(method, url, error, status, redirect_location),\n        )\n    \n        new_retry = self.new(\n            total=total,\n            connect=connect,\n            read=read,\n            redirect=redirect,\n            status=status_count,\n            other=other,\n            history=history,\n        )\n    \n        if new_retry.is_exhausted():\n            reason = error or ResponseError(cause)\n>           raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\nE           urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:519: MaxRetryError\n\nDuring handling of the above exception, another exception occurred:\n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n>               response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n\nllmisvc/test_llm_inference_service.py:1095: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/v1/completions'\n\n    def post_with_retry(\n        url: str,\n        *,\n        headers: Dict = None,\n        json_data: Union[Dict, List] = None,\n        data: Union[str, bytes] = None,\n        stream: bool = False,\n        timeout: float = None,\n        total_retries: int = DEFAULT_RETRY_TOTAL,\n        backoff_factor: float = DEFAULT_RETRY_BACKOFF_FACTOR,\n        retry_status_codes=DEFAULT_RETRY_STATUS_CODES,\n    ) -> requests.Response:\n        \"\"\"\n        Send POST request with retries for transient HTTP and network failures.\n        \"\"\"\n        if json_data is not None and data is not None:\n            raise ValueError(\"Only one of json_data or data can be provided.\")\n    \n        with _retry_session(\n            [\"POST\"], total_retries, backoff_factor, retry_status_codes\n        ) as session:\n>           return session.post(\n                url,\n                json=json_data,\n                data=data,\n                headers=headers,\n                stream=stream,\n                timeout=timeout,\n            )\n\ncommon/http_retry.py:70: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7fcebc12c9d0>\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/v1/completions'\ndata = None\njson = {'max_tokens': 20, 'model': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'prompt': 'KServe is a'}\nkwargs = {'headers': {'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1'}, 'stream': False, 'timeout': 60}\n\n    def post(self, url, data=None, json=None, **kwargs):\n        r\"\"\"Sends a POST request. Returns :class:`Response` object.\n    \n        :param url: URL for the new :class:`Request` object.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the :class:`Request`.\n        :param \\*\\*kwargs: Optional arguments that ``request`` takes.\n        :rtype: requests.Response\n        \"\"\"\n    \n>       return self.request(\"POST\", url, data=data, json=json, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:637: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7fcebc12c9d0>, method = 'POST'\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/v1/completions'\nparams = None, data = None\nheaders = {'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1'}\ncookies = None, files = None, auth = None, timeout = 60, allow_redirects = True\nproxies = {}, hooks = None, stream = False, verify = None, cert = None\njson = {'max_tokens': 20, 'model': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1', 'prompt': 'KServe is a'}\n\n    def request(\n        self,\n        method,\n        url,\n        params=None,\n        data=None,\n        headers=None,\n        cookies=None,\n        files=None,\n        auth=None,\n        timeout=None,\n        allow_redirects=True,\n        proxies=None,\n        hooks=None,\n        stream=None,\n        verify=None,\n        cert=None,\n        json=None,\n    ):\n        \"\"\"Constructs a :class:`Request <Request>`, prepares it and sends it.\n        Returns :class:`Response <Response>` object.\n    \n        :param method: method for the new :class:`Request` object.\n        :param url: URL for the new :class:`Request` object.\n        :param params: (optional) Dictionary or bytes to be sent in the query\n            string for the :class:`Request`.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the\n            :class:`Request`.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the\n            :class:`Request`.\n        :param cookies: (optional) Dict or CookieJar object to send with the\n            :class:`Request`.\n        :param files: (optional) Dictionary of ``'filename': file-like-objects``\n            for multipart encoding upload.\n        :param auth: (optional) Auth tuple or callable to enable\n            Basic/Digest/Custom HTTP Auth.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple\n        :param allow_redirects: (optional) Set to True by default.\n        :type allow_redirects: bool\n        :param proxies: (optional) Dictionary mapping protocol or protocol and\n            hostname to the URL of the proxy.\n        :param hooks: (optional) Dictionary mapping hook name to one event or\n            list of events, event must be callable.\n        :param stream: (optional) whether to immediately download the response\n            content. Defaults to ``False``.\n        :param verify: (optional) Either a boolean, in which case it controls whether we verify\n            the server's TLS certificate, or a string, in which case it must be a path\n            to a CA bundle to use. Defaults to ``True``. When set to\n            ``False``, requests will accept any TLS certificate presented by\n            the server, and will ignore hostname mismatches and/or expired\n            certificates, which will make your application vulnerable to\n            man-in-the-middle (MitM) attacks. Setting verify to ``False``\n            may be useful during local development or testing.\n        :param cert: (optional) if String, path to ssl client cert file (.pem).\n            If Tuple, ('cert', 'key') pair.\n        :rtype: requests.Response\n        \"\"\"\n        # Create the Request.\n        req = Request(\n            method=method.upper(),\n            url=url,\n            headers=headers,\n            files=files,\n            data=data or {},\n            json=json,\n            params=params or {},\n            auth=auth,\n            cookies=cookies,\n            hooks=hooks,\n        )\n        prep = self.prepare_request(req)\n    \n        proxies = proxies or {}\n    \n        settings = self.merge_environment_settings(\n            prep.url, proxies, stream, verify, cert\n        )\n    \n        # Send the request.\n        send_kwargs = {\n            \"timeout\": timeout,\n            \"allow_redirects\": allow_redirects,\n        }\n        send_kwargs.update(settings)\n>       resp = self.send(prep, **send_kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:589: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7fcebc12c9d0>\nrequest = <PreparedRequest [POST]>\nkwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': 60, ...}\nallow_redirects = True, stream = False, hooks = {'response': []}\nadapter = <requests.adapters.HTTPAdapter object at 0x7fcebc06f350>\nstart = 1783425801.028785\n\n    def send(self, request, **kwargs):\n        \"\"\"Send a given PreparedRequest.\n    \n        :rtype: requests.Response\n        \"\"\"\n        # Set defaults that the hooks can utilize to ensure they always have\n        # the correct parameters to reproduce the previous request.\n        kwargs.setdefault(\"stream\", self.stream)\n        kwargs.setdefault(\"verify\", self.verify)\n        kwargs.setdefault(\"cert\", self.cert)\n        if \"proxies\" not in kwargs:\n            kwargs[\"proxies\"] = resolve_proxies(request, self.proxies, self.trust_env)\n    \n        # It's possible that users might accidentally send a Request object.\n        # Guard against that specific failure case.\n        if isinstance(request, Request):\n            raise ValueError(\"You can only send PreparedRequests.\")\n    \n        # Set up variables needed for resolve_redirects and dispatching of hooks\n        allow_redirects = kwargs.pop(\"allow_redirects\", True)\n        stream = kwargs.get(\"stream\")\n        hooks = request.hooks\n    \n        # Get the appropriate adapter to use\n        adapter = self.get_adapter(url=request.url)\n    \n        # Start time (approximately) of the request\n        start = preferred_clock()\n    \n        # Send the request\n>       r = adapter.send(request, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:703: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.adapters.HTTPAdapter object at 0x7fcebc06f350>\nrequest = <PreparedRequest [POST]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n            resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n    \n        except (ProtocolError, OSError) as err:\n            raise ConnectionError(err, request=request)\n    \n        except MaxRetryError as e:\n            if isinstance(e.reason, ConnectTimeoutError):\n                # TODO: Remove this in 3.0.0: see #2811\n                if not isinstance(e.reason, NewConnectionError):\n                    raise ConnectTimeout(e, request=request)\n    \n            if isinstance(e.reason, ResponseError):\n                raise RetryError(e, request=request)\n    \n            if isinstance(e.reason, _ProxyError):\n                raise ProxyError(e, request=request)\n    \n            if isinstance(e.reason, _SSLError):\n                # This branch is for urllib3 v1.22 and later.\n                raise SSLError(e, request=request)\n    \n>           raise ConnectionError(e, request=request)\nE           requests.exceptions.ConnectionError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:700: ConnectionError\n\nThe above exception was the direct cause of the following exception:\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m-with-lora-hf'], prompt='KServe is a', ...opt-125m-with-lora-hf-a7886ead'}]},\n 'status': None}, model_name='publishers/kserve-ci-e2e-test/models/lora-adapter-1')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.asyncio(loop_scope=\"session\")\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-gateway-ref\",\n                        \"router-with-managed-route\",\n                        \"model-fb-opt-125m\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"custom-route-timeout-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"router-with-refs-test\",\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                            routes=[ROUTER_ROUTES[0], ROUTER_ROUTES[1]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\"router-managed\", \"workload-pd-cpu\", \"model-fb-opt-125m\"],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"custom-route-timeout-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"router-with-refs-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                    expected_gateway=ROUTER_GATEWAYS[1],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[1]],\n                            routes=[ROUTER_ROUTES[2], ROUTER_ROUTES[3]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-dp-ep-gpu\",\n                        \"workload-dp-ep-prefill-gpu\",\n                        \"model-deepseek-v2-lite\",\n                    ],\n                    prompt=\"Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically \"\n                    \"where the compute plane (P) and the data plane (D) are independently deployed and managed for a \"\n                    \"geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the \"\n                    \"fundamental challenges of network latency and data consistency, elaborate on the advanced \"\n                    \"considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: \"\n                    \"How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to \"\n                    \"evolve to support optimal performance and minimize inter-plane communication overhead, especially for \"\n                    \"synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically \"\n                    \"optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: \"\n                    \"Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) \"\n                    \"and their applicability in balancing performance and data integrity across a globally distributed data plane. \"\n                    \"Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, \"\n                    \"intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. \"\n                    \"3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently \"\n                    \"manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, \"\n                    \"cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). \"\n                    \"Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on \"\n                    \"workload patterns and data locality, potentially involving live migration strategies. \"\n                    \"4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter \"\n                    \"challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), \"\n                    \"fine-grained access control to data at rest and in motion, and identity management across disaggregated \"\n                    \"components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) \"\n                    \"concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: \"\n                    \"Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and \"\n                    \"data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) \"\n                    \"would be essential? How would incident response and troubleshooting differ in this disaggregated environment \"\n                    \"compared to traditional integrated systems? Consider the challenges of pinpointing root causes across \"\n                    \"independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries \"\n                    \"or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) \"\n                    \"where the benefits of P/D disaggregation would strongly outweigh its complexities. \"\n                    \"Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions \"\n                    \"directly interacting with object storage, in-memory disaggregation) that could further drive or \"\n                    \"transform P/D disaggregation in cloud computing.\",\n                    max_tokens=2000,\n                ),\n                marks=[\n                    pytest.mark.cluster_gpu,\n                    pytest.mark.cluster_nvidia,\n                    pytest.mark.cluster_nvidia_roce,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-no-scheduler\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"What is KServe?\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.no_scheduler,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, \"\n                    \"but without the resources requirements for DP+EP (GPUs and ROCe/IB).\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],\n            ),\n            # Scheduler config tests\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-inline-config\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-inline-config-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Chat completions endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                        \"model-qwen2.5-0.5b\",\n                    ],\n                    model_name=\"Qwen/Qwen2.5-0.5B-Instruct\",\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-configmap-ref\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-configmap-ref-test\",\n                    before_test=[create_scheduler_configmap],\n                    after_test=[delete_scheduler_configmap],\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-replicas\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-ha-replicas-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-custom-template\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-custom-template-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Scheduler v0.6 \u2192 v0.7 migration tests.\n            # Deploy v0.6-style configs and verify the controller migrates them\n            # so the v0.7 scheduler boots successfully.\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-pd-config-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-pd-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-nonzero-threshold-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-threshold-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Precise prefix KV cache routing test\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-precise-prefix-cache-inline-config\",\n                        \"workload-llmd-simulator-kvcache\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"precise-prefix-cache-test\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Models endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=create_response_assertion(with_field=\"data\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/completions\",\n                            prompt=\"KServe is a\",\n                            payload_formatter=completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/chat/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/chat/completions\",\n                            prompt=\"What is KServe?\",\n                            payload_formatter=chat_completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 LoRA adapter\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    model_name=f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\"\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/models (base + LoRA)\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=assert_models_contains(\n                        \"facebook/opt-125m\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                        \"lora-adapter-1\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # PVC storage tests -- validate direct PVC volume mount with real vLLM serving\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_multi_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_llm_inference_service(test_case: TestCase):  # noqa: F811\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        if not test_case.llm_service.metadata.annotations:\n            test_case.llm_service.metadata.annotations = {}\n    \n        test_case.llm_service.metadata.annotations[\n            \"security.opendatahub.io/enable-auth\"\n        ] = \"false\"\n        prefix = test_case.log_prefix\n    \n        test_failed = False\n        try:\n            print(f\"{prefix} Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n            print(f\"{prefix} Waiting for LLMInferenceService {service_name} to be ready\")\n            wait_for_llm_isvc_ready(\n                kserve_client, test_case.llm_service, test_case.wait_timeout\n            )\n            print(f\"{prefix} Waiting for model response from {service_name}\")\n>           wait_for_model_response(\n                kserve_client,\n                test_case,\n                test_case.wait_timeout,\n                extra_headers=test_case.extra_headers,\n            )\n\nllmisvc/test_llm_inference_service.py:816: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (<kserve.api.kserve_client.KServeClient object at 0x7fcebc2411d0>, TestCase(base_refs=['router-managed', 'workload-sin...5m-with-lora-hf-a7886ead'}]},\n 'status': None}, model_name='publishers/kserve-ci-e2e-test/models/lora-adapter-1'), 900)\nkwargs = {'extra_headers': {'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1'}}\nfunc_name = 'wait_for_model_response'\ntimestamp_start = '2026-07-07T12:03:20.965134', start_time = 1783425800.9654002\nduration = 904.7197306156158, timestamp_end = '2026-07-07T12:18:25.685133'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fcebc2411d0>\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m-with-lora-hf'], prompt='KServe is a', ...opt-125m-with-lora-hf-a7886ead'}]},\n 'status': None}, model_name='publishers/kserve-ci-e2e-test/models/lora-adapter-1')\ntimeout_seconds = 900\nextra_headers = {'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/lora-adapter-1'}\n\n    @log_execution\n    def wait_for_model_response(\n        kserve_client: KServeClient,\n        test_case: TestCase,  # noqa: F811\n        timeout_seconds: int = 900,\n        extra_headers: Optional[Dict[str, str]] = None,\n    ) -> str:\n        def get_successful_response():\n            try:\n                if test_case.url_getter:\n                    service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n                else:\n                    service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n            except Exception as e:\n                raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n            model_url = service_url + test_case.endpoint\n    \n            headers = {\"Content-Type\": \"application/json\"}\n            if extra_headers:\n                headers.update(extra_headers)\n    \n            if test_case.payload_formatter is not None:\n                test_payload = test_case.payload_formatter(test_case)\n            elif test_case.prompt is not None:\n                test_payload = {\n                    \"model\": test_case.model_name\n                    if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                    else extra_headers[MODEL_ROUTING_HEADER],\n                    \"prompt\": test_case.prompt,\n                    \"max_tokens\": test_case.max_tokens,\n                }\n            else:\n                test_payload = None\n    \n            logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n            try:\n                if test_payload is not None:\n                    response = post_with_retry(\n                        model_url,\n                        headers=headers,\n                        json_data=test_payload,\n                        timeout=test_case.response_timeout,\n                    )\n                else:\n                    response = get_with_retry(\n                        model_url,\n                        headers=headers,\n                        timeout=test_case.response_timeout,\n                    )\n            except Exception as e:\n                logger.error(f\"\u274c Failed to call model: {e}\")\n                raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\n    \n            logger.info(f\"Model response is {response.status_code}: {response.text[:500]}\")\n    \n            if 200 <= response.status_code < 300:\n                return response\n            raise AssertionError(\n                f\"Service returned {response.status_code}: {response.text}\"\n            )\n    \n>       response = wait_for(get_successful_response, timeout=timeout_seconds, interval=5.0)\n\nllmisvc/test_llm_inference_service.py:1119: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_model_response.<locals>.get_successful_response at 0x7fcebc1956c0>\ntimeout = 900, interval = 5.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n                response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n            else:\n                response = get_with_retry(\n                    model_url,\n                    headers=headers,\n                    timeout=test_case.response_timeout,\n                )\n        except Exception as e:\n            logger.error(f\"\u274c Failed to call model: {e}\")\n>           raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\nE           AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\nllmisvc/test_llm_inference_service.py:1109: AssertionError"}, "teardown": {"duration": 0.0012892890008515678, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-with-gateway-ref-router-with-managed-route-model-fb-opt-125m-workload-llmd-simulator]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-with-gateway-ref-router-with-managed-route-model-fb-opt-125m-workload-llmd-simulator]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "custom_gateway", "__wrapped__", "pytestmark", "router-with-gateway-ref-router-with-managed-route-model-fb-opt-125m-workload-llmd-simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.5662274839996826, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 44.4467274339986, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0015444359996763524, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "failed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.40589072900183965, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 1041.5348615660005, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))"}, "traceback": [{"path": "llmisvc/test_llm_inference_service.py", "lineno": 816, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1119, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953faeed90>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953faeed90>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n>           response = conn.getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:534: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953faeed90>\n\n    def getresponse(  # type: ignore[override]\n        self,\n    ) -> HTTPResponse:\n        \"\"\"\n        Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.\n    \n        If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.\n        \"\"\"\n        # Raise the same error as http.client.HTTPConnection\n        if self._response_options is None:\n            raise ResponseNotReady()\n    \n        # Reset this attribute for being used again.\n        resp_options = self._response_options\n        self._response_options = None\n    \n        # Since the connection's timeout value may have been updated\n        # we need to set the timeout on the socket.\n        self.sock.settimeout(self.timeout)\n    \n        # This is needed here to avoid circular import errors\n        from .response import HTTPResponse\n    \n        # Save a reference to the shutdown function before ownership is passed\n        # to httplib_response\n        # TODO should we implement it everywhere?\n        _shutdown = getattr(self.sock, \"shutdown\", None)\n    \n        # Get the response from http.client.HTTPConnection\n>       httplib_response = super().getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:571: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953faeed90>\n\n    def getresponse(self):\n        \"\"\"Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an\n        instance of HTTPResponse or of whatever object is returned by\n        the response_class variable.\n    \n        If a request has not been sent or if a previous response has\n        not be handled, ResponseNotReady is raised.  If the HTTP\n        response indicates that the connection should be closed, then\n        it will be closed before the response is returned.  When the\n        connection is closed, the underlying socket is closed.\n        \"\"\"\n    \n        # if a prior response has been completed, then forget about it.\n        if self.__response and self.__response.isclosed():\n            self.__response = None\n    \n        # if a prior response exists, then it must be completed (otherwise, we\n        # cannot read this response's header to determine the connection-close\n        # behavior)\n        #\n        # note: if a prior response existed, but was connection-close, then the\n        # socket and response were made independent of this HTTPConnection\n        # object since a new request requires that we open a whole new\n        # connection\n        #\n        # this means the prior response had one of two states:\n        #   1) will_close: this connection was reset and the prior socket and\n        #                  response operate independently\n        #   2) persistent: the response was retained and we await its\n        #                  isclosed() status to become true.\n        #\n        if self.__state != _CS_REQ_SENT or self.__response:\n            raise ResponseNotReady(self.__state)\n    \n        if self.debuglevel > 0:\n            response = self.response_class(self.sock, self.debuglevel,\n                                           method=self._method)\n        else:\n            response = self.response_class(self.sock, method=self._method)\n    \n        try:\n            try:\n>               response.begin()\n\n/usr/lib64/python3.11/http/client.py:1395: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7f953feae410>\n\n    def begin(self):\n        if self.headers is not None:\n            # we've already started reading the response\n            return\n    \n        # read until we get a non-100 response\n        while True:\n>           version, status, reason = self._read_status()\n\n/usr/lib64/python3.11/http/client.py:325: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7f953feae410>\n\n    def _read_status(self):\n>       line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n\n/usr/lib64/python3.11/http/client.py:286: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <socket.SocketIO object at 0x7f953feacd60>\nb = <memory at 0x7f953f9f0d00>\n\n    def readinto(self, b):\n        \"\"\"Read up to len(b) bytes into the writable buffer *b* and return\n        the number of bytes read.  If the socket is non-blocking and no bytes\n        are available, None is returned.\n    \n        If *b* is non-empty, a 0 return value indicates that the connection\n        was shutdown at the other end.\n        \"\"\"\n        self._checkClosed()\n        self._checkReadable()\n        if self._timeout_occurred:\n            raise OSError(\"cannot read from timed out object\")\n        while True:\n            try:\n>               return self._sock.recv_into(b)\nE               TimeoutError: timed out\n\n/usr/lib64/python3.11/socket.py:718: TimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n>           response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:787: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953faeed90>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953faeed90>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n            response = conn.getresponse()\n        except (BaseSSLError, OSError) as e:\n>           self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:536: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nerr = TimeoutError('timed out')\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\ntimeout_value = 60\n\n    def _raise_timeout(\n        self,\n        err: BaseSSLError | OSError | SocketTimeout,\n        url: str,\n        timeout_value: _TYPE_TIMEOUT | None,\n    ) -> None:\n        \"\"\"Is the error actually a timeout? Will raise a ReadTimeout or pass\"\"\"\n    \n        if isinstance(err, SocketTimeout):\n>           raise ReadTimeoutError(\n                self, url, f\"Read timed out. (read timeout={timeout_value})\"\n            ) from err\nE           urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <requests.adapters.HTTPAdapter object at 0x7f953f838090>\nrequest = <PreparedRequest [POST]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n>           resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:667: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=7, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=6, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=5, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=4, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=3, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=2, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=1, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n>           retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:841: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nresponse = None\nerror = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\n_pool = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953f884210>\n_stacktrace = <traceback object at 0x7f953faed700>\n\n    def increment(\n        self,\n        method: str | None = None,\n        url: str | None = None,\n        response: BaseHTTPResponse | None = None,\n        error: Exception | None = None,\n        _pool: ConnectionPool | None = None,\n        _stacktrace: TracebackType | None = None,\n    ) -> Self:\n        \"\"\"Return a new Retry object with incremented retry counters.\n    \n        :param response: A response object, or None, if the server did not\n            return a response.\n        :type response: :class:`~urllib3.response.BaseHTTPResponse`\n        :param Exception error: An error encountered during the request, or\n            None if the response was received successfully.\n    \n        :return: A new ``Retry`` object.\n        \"\"\"\n        if self.total is False and error:\n            # Disabled, indicate to re-raise the error.\n            raise reraise(type(error), error, _stacktrace)\n    \n        total = self.total\n        if total is not None:\n            total -= 1\n    \n        connect = self.connect\n        read = self.read\n        redirect = self.redirect\n        status_count = self.status\n        other = self.other\n        cause = \"unknown\"\n        status = None\n        redirect_location = None\n    \n        if error and self._is_connection_error(error):\n            # Connect retry?\n            if connect is False:\n                raise reraise(type(error), error, _stacktrace)\n            elif connect is not None:\n                connect -= 1\n    \n        elif error and self._is_read_error(error):\n            # Read retry?\n            if read is False or method is None or not self._is_method_retryable(method):\n                raise reraise(type(error), error, _stacktrace)\n            elif read is not None:\n                read -= 1\n    \n        elif error:\n            # Other retry?\n            if other is not None:\n                other -= 1\n    \n        elif response and response.get_redirect_location():\n            # Redirect retry?\n            if redirect is not None:\n                redirect -= 1\n            cause = \"too many redirects\"\n            response_redirect_location = response.get_redirect_location()\n            if response_redirect_location:\n                redirect_location = response_redirect_location\n            status = response.status\n    \n        else:\n            # Incrementing because of a server error like a 500 in\n            # status_forcelist and the given method is in the allowed_methods\n            cause = ResponseError.GENERIC_ERROR\n            if response and response.status:\n                if status_count is not None:\n                    status_count -= 1\n                cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)\n                status = response.status\n    \n        history = self.history + (\n            RequestHistory(method, url, error, status, redirect_location),\n        )\n    \n        new_retry = self.new(\n            total=total,\n            connect=connect,\n            read=read,\n            redirect=redirect,\n            status=status_count,\n            other=other,\n            history=history,\n        )\n    \n        if new_retry.is_exhausted():\n            reason = error or ResponseError(cause)\n>           raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\nE           urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:519: MaxRetryError\n\nDuring handling of the above exception, another exception occurred:\n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n>               response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n\nllmisvc/test_llm_inference_service.py:1095: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\n\n    def post_with_retry(\n        url: str,\n        *,\n        headers: Dict = None,\n        json_data: Union[Dict, List] = None,\n        data: Union[str, bytes] = None,\n        stream: bool = False,\n        timeout: float = None,\n        total_retries: int = DEFAULT_RETRY_TOTAL,\n        backoff_factor: float = DEFAULT_RETRY_BACKOFF_FACTOR,\n        retry_status_codes=DEFAULT_RETRY_STATUS_CODES,\n    ) -> requests.Response:\n        \"\"\"\n        Send POST request with retries for transient HTTP and network failures.\n        \"\"\"\n        if json_data is not None and data is not None:\n            raise ValueError(\"Only one of json_data or data can be provided.\")\n    \n        with _retry_session(\n            [\"POST\"], total_retries, backoff_factor, retry_status_codes\n        ) as session:\n>           return session.post(\n                url,\n                json=json_data,\n                data=data,\n                headers=headers,\n                stream=stream,\n                timeout=timeout,\n            )\n\ncommon/http_retry.py:70: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7f953f838350>\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\ndata = None\njson = {'max_tokens': 20, 'model': 'facebook/opt-125m', 'prompt': 'KServe is a'}\nkwargs = {'headers': {'Content-Type': 'application/json'}, 'stream': False, 'timeout': 60}\n\n    def post(self, url, data=None, json=None, **kwargs):\n        r\"\"\"Sends a POST request. Returns :class:`Response` object.\n    \n        :param url: URL for the new :class:`Request` object.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the :class:`Request`.\n        :param \\*\\*kwargs: Optional arguments that ``request`` takes.\n        :rtype: requests.Response\n        \"\"\"\n    \n>       return self.request(\"POST\", url, data=data, json=json, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:637: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7f953f838350>, method = 'POST'\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions'\nparams = None, data = None, headers = {'Content-Type': 'application/json'}\ncookies = None, files = None, auth = None, timeout = 60, allow_redirects = True\nproxies = {}, hooks = None, stream = False, verify = None, cert = None\njson = {'max_tokens': 20, 'model': 'facebook/opt-125m', 'prompt': 'KServe is a'}\n\n    def request(\n        self,\n        method,\n        url,\n        params=None,\n        data=None,\n        headers=None,\n        cookies=None,\n        files=None,\n        auth=None,\n        timeout=None,\n        allow_redirects=True,\n        proxies=None,\n        hooks=None,\n        stream=None,\n        verify=None,\n        cert=None,\n        json=None,\n    ):\n        \"\"\"Constructs a :class:`Request <Request>`, prepares it and sends it.\n        Returns :class:`Response <Response>` object.\n    \n        :param method: method for the new :class:`Request` object.\n        :param url: URL for the new :class:`Request` object.\n        :param params: (optional) Dictionary or bytes to be sent in the query\n            string for the :class:`Request`.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the\n            :class:`Request`.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the\n            :class:`Request`.\n        :param cookies: (optional) Dict or CookieJar object to send with the\n            :class:`Request`.\n        :param files: (optional) Dictionary of ``'filename': file-like-objects``\n            for multipart encoding upload.\n        :param auth: (optional) Auth tuple or callable to enable\n            Basic/Digest/Custom HTTP Auth.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple\n        :param allow_redirects: (optional) Set to True by default.\n        :type allow_redirects: bool\n        :param proxies: (optional) Dictionary mapping protocol or protocol and\n            hostname to the URL of the proxy.\n        :param hooks: (optional) Dictionary mapping hook name to one event or\n            list of events, event must be callable.\n        :param stream: (optional) whether to immediately download the response\n            content. Defaults to ``False``.\n        :param verify: (optional) Either a boolean, in which case it controls whether we verify\n            the server's TLS certificate, or a string, in which case it must be a path\n            to a CA bundle to use. Defaults to ``True``. When set to\n            ``False``, requests will accept any TLS certificate presented by\n            the server, and will ignore hostname mismatches and/or expired\n            certificates, which will make your application vulnerable to\n            man-in-the-middle (MitM) attacks. Setting verify to ``False``\n            may be useful during local development or testing.\n        :param cert: (optional) if String, path to ssl client cert file (.pem).\n            If Tuple, ('cert', 'key') pair.\n        :rtype: requests.Response\n        \"\"\"\n        # Create the Request.\n        req = Request(\n            method=method.upper(),\n            url=url,\n            headers=headers,\n            files=files,\n            data=data or {},\n            json=json,\n            params=params or {},\n            auth=auth,\n            cookies=cookies,\n            hooks=hooks,\n        )\n        prep = self.prepare_request(req)\n    \n        proxies = proxies or {}\n    \n        settings = self.merge_environment_settings(\n            prep.url, proxies, stream, verify, cert\n        )\n    \n        # Send the request.\n        send_kwargs = {\n            \"timeout\": timeout,\n            \"allow_redirects\": allow_redirects,\n        }\n        send_kwargs.update(settings)\n>       resp = self.send(prep, **send_kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:589: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7f953f838350>\nrequest = <PreparedRequest [POST]>\nkwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': 60, ...}\nallow_redirects = True, stream = False, hooks = {'response': []}\nadapter = <requests.adapters.HTTPAdapter object at 0x7f953f838090>\nstart = 1783425935.4143379\n\n    def send(self, request, **kwargs):\n        \"\"\"Send a given PreparedRequest.\n    \n        :rtype: requests.Response\n        \"\"\"\n        # Set defaults that the hooks can utilize to ensure they always have\n        # the correct parameters to reproduce the previous request.\n        kwargs.setdefault(\"stream\", self.stream)\n        kwargs.setdefault(\"verify\", self.verify)\n        kwargs.setdefault(\"cert\", self.cert)\n        if \"proxies\" not in kwargs:\n            kwargs[\"proxies\"] = resolve_proxies(request, self.proxies, self.trust_env)\n    \n        # It's possible that users might accidentally send a Request object.\n        # Guard against that specific failure case.\n        if isinstance(request, Request):\n            raise ValueError(\"You can only send PreparedRequests.\")\n    \n        # Set up variables needed for resolve_redirects and dispatching of hooks\n        allow_redirects = kwargs.pop(\"allow_redirects\", True)\n        stream = kwargs.get(\"stream\")\n        hooks = request.hooks\n    \n        # Get the appropriate adapter to use\n        adapter = self.get_adapter(url=request.url)\n    \n        # Start time (approximately) of the request\n        start = preferred_clock()\n    \n        # Send the request\n>       r = adapter.send(request, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:703: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.adapters.HTTPAdapter object at 0x7f953f838090>\nrequest = <PreparedRequest [POST]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n            resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n    \n        except (ProtocolError, OSError) as err:\n            raise ConnectionError(err, request=request)\n    \n        except MaxRetryError as e:\n            if isinstance(e.reason, ConnectTimeoutError):\n                # TODO: Remove this in 3.0.0: see #2811\n                if not isinstance(e.reason, NewConnectionError):\n                    raise ConnectTimeout(e, request=request)\n    \n            if isinstance(e.reason, ResponseError):\n                raise RetryError(e, request=request)\n    \n            if isinstance(e.reason, _ProxyError):\n                raise ProxyError(e, request=request)\n    \n            if isinstance(e.reason, _SSLError):\n                # This branch is for urllib3 v1.22 and later.\n                raise SSLError(e, request=request)\n    \n>           raise ConnectionError(e, request=request)\nE           requests.exceptions.ConnectionError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:700: ConnectionError\n\nThe above exception was the direct cause of the following exception:\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m'], prompt='KServe is a', service_name=...              {'name': 'model-fb-opt-125m-llmisvc-model-2a331009'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.asyncio(loop_scope=\"session\")\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-gateway-ref\",\n                        \"router-with-managed-route\",\n                        \"model-fb-opt-125m\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"custom-route-timeout-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"router-with-refs-test\",\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                            routes=[ROUTER_ROUTES[0], ROUTER_ROUTES[1]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\"router-managed\", \"workload-pd-cpu\", \"model-fb-opt-125m\"],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"custom-route-timeout-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"router-with-refs-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                    expected_gateway=ROUTER_GATEWAYS[1],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[1]],\n                            routes=[ROUTER_ROUTES[2], ROUTER_ROUTES[3]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-dp-ep-gpu\",\n                        \"workload-dp-ep-prefill-gpu\",\n                        \"model-deepseek-v2-lite\",\n                    ],\n                    prompt=\"Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically \"\n                    \"where the compute plane (P) and the data plane (D) are independently deployed and managed for a \"\n                    \"geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the \"\n                    \"fundamental challenges of network latency and data consistency, elaborate on the advanced \"\n                    \"considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: \"\n                    \"How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to \"\n                    \"evolve to support optimal performance and minimize inter-plane communication overhead, especially for \"\n                    \"synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically \"\n                    \"optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: \"\n                    \"Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) \"\n                    \"and their applicability in balancing performance and data integrity across a globally distributed data plane. \"\n                    \"Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, \"\n                    \"intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. \"\n                    \"3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently \"\n                    \"manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, \"\n                    \"cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). \"\n                    \"Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on \"\n                    \"workload patterns and data locality, potentially involving live migration strategies. \"\n                    \"4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter \"\n                    \"challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), \"\n                    \"fine-grained access control to data at rest and in motion, and identity management across disaggregated \"\n                    \"components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) \"\n                    \"concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: \"\n                    \"Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and \"\n                    \"data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) \"\n                    \"would be essential? How would incident response and troubleshooting differ in this disaggregated environment \"\n                    \"compared to traditional integrated systems? Consider the challenges of pinpointing root causes across \"\n                    \"independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries \"\n                    \"or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) \"\n                    \"where the benefits of P/D disaggregation would strongly outweigh its complexities. \"\n                    \"Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions \"\n                    \"directly interacting with object storage, in-memory disaggregation) that could further drive or \"\n                    \"transform P/D disaggregation in cloud computing.\",\n                    max_tokens=2000,\n                ),\n                marks=[\n                    pytest.mark.cluster_gpu,\n                    pytest.mark.cluster_nvidia,\n                    pytest.mark.cluster_nvidia_roce,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-no-scheduler\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"What is KServe?\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.no_scheduler,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, \"\n                    \"but without the resources requirements for DP+EP (GPUs and ROCe/IB).\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],\n            ),\n            # Scheduler config tests\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-inline-config\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-inline-config-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Chat completions endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                        \"model-qwen2.5-0.5b\",\n                    ],\n                    model_name=\"Qwen/Qwen2.5-0.5B-Instruct\",\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-configmap-ref\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-configmap-ref-test\",\n                    before_test=[create_scheduler_configmap],\n                    after_test=[delete_scheduler_configmap],\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-replicas\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-ha-replicas-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-custom-template\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-custom-template-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Scheduler v0.6 \u2192 v0.7 migration tests.\n            # Deploy v0.6-style configs and verify the controller migrates them\n            # so the v0.7 scheduler boots successfully.\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-pd-config-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-pd-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-nonzero-threshold-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-threshold-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Precise prefix KV cache routing test\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-precise-prefix-cache-inline-config\",\n                        \"workload-llmd-simulator-kvcache\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"precise-prefix-cache-test\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Models endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=create_response_assertion(with_field=\"data\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/completions\",\n                            prompt=\"KServe is a\",\n                            payload_formatter=completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/chat/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/chat/completions\",\n                            prompt=\"What is KServe?\",\n                            payload_formatter=chat_completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 LoRA adapter\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    model_name=f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\"\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/models (base + LoRA)\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=assert_models_contains(\n                        \"facebook/opt-125m\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                        \"lora-adapter-1\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # PVC storage tests -- validate direct PVC volume mount with real vLLM serving\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_multi_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_llm_inference_service(test_case: TestCase):  # noqa: F811\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        if not test_case.llm_service.metadata.annotations:\n            test_case.llm_service.metadata.annotations = {}\n    \n        test_case.llm_service.metadata.annotations[\n            \"security.opendatahub.io/enable-auth\"\n        ] = \"false\"\n        prefix = test_case.log_prefix\n    \n        test_failed = False\n        try:\n            print(f\"{prefix} Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n            print(f\"{prefix} Waiting for LLMInferenceService {service_name} to be ready\")\n            wait_for_llm_isvc_ready(\n                kserve_client, test_case.llm_service, test_case.wait_timeout\n            )\n            print(f\"{prefix} Waiting for model response from {service_name}\")\n>           wait_for_model_response(\n                kserve_client,\n                test_case,\n                test_case.wait_timeout,\n                extra_headers=test_case.extra_headers,\n            )\n\nllmisvc/test_llm_inference_service.py:816: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (<kserve.api.kserve_client.KServeClient object at 0x7f953fa2fad0>, TestCase(base_refs=['router-managed', 'workload-sin...        {'name': 'model-fb-opt-125m-llmisvc-model-2a331009'}]},\n 'status': None}, model_name='facebook/opt-125m'), 900)\nkwargs = {'extra_headers': None}, func_name = 'wait_for_model_response'\ntimestamp_start = '2026-07-07T12:05:35.404298', start_time = 1783425935.4046109\nduration = 904.7419965267181, timestamp_end = '2026-07-07T12:20:40.146610'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f953fa2fad0>\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m'], prompt='KServe is a', service_name=...              {'name': 'model-fb-opt-125m-llmisvc-model-2a331009'}]},\n 'status': None}, model_name='facebook/opt-125m')\ntimeout_seconds = 900, extra_headers = None\n\n    @log_execution\n    def wait_for_model_response(\n        kserve_client: KServeClient,\n        test_case: TestCase,  # noqa: F811\n        timeout_seconds: int = 900,\n        extra_headers: Optional[Dict[str, str]] = None,\n    ) -> str:\n        def get_successful_response():\n            try:\n                if test_case.url_getter:\n                    service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n                else:\n                    service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n            except Exception as e:\n                raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n            model_url = service_url + test_case.endpoint\n    \n            headers = {\"Content-Type\": \"application/json\"}\n            if extra_headers:\n                headers.update(extra_headers)\n    \n            if test_case.payload_formatter is not None:\n                test_payload = test_case.payload_formatter(test_case)\n            elif test_case.prompt is not None:\n                test_payload = {\n                    \"model\": test_case.model_name\n                    if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                    else extra_headers[MODEL_ROUTING_HEADER],\n                    \"prompt\": test_case.prompt,\n                    \"max_tokens\": test_case.max_tokens,\n                }\n            else:\n                test_payload = None\n    \n            logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n            try:\n                if test_payload is not None:\n                    response = post_with_retry(\n                        model_url,\n                        headers=headers,\n                        json_data=test_payload,\n                        timeout=test_case.response_timeout,\n                    )\n                else:\n                    response = get_with_retry(\n                        model_url,\n                        headers=headers,\n                        timeout=test_case.response_timeout,\n                    )\n            except Exception as e:\n                logger.error(f\"\u274c Failed to call model: {e}\")\n                raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\n    \n            logger.info(f\"Model response is {response.status_code}: {response.text[:500]}\")\n    \n            if 200 <= response.status_code < 300:\n                return response\n            raise AssertionError(\n                f\"Service returned {response.status_code}: {response.text}\"\n            )\n    \n>       response = wait_for(get_successful_response, timeout=timeout_seconds, interval=5.0)\n\nllmisvc/test_llm_inference_service.py:1119: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_model_response.<locals>.get_successful_response at 0x7f953fae7ec0>\ntimeout = 900, interval = 5.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n                response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n            else:\n                response = get_with_retry(\n                    model_url,\n                    headers=headers,\n                    timeout=test_case.response_timeout,\n                )\n        except Exception as e:\n            logger.error(f\"\u274c Failed to call model: {e}\")\n>           raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\nE           AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/llmisvc-model-fb-opt-125m-route-87882a8e/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\nllmisvc/test_llm_inference_service.py:1109: AssertionError"}, "teardown": {"duration": 0.001918085999932373, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1]", "lineno": 244, "outcome": "failed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "model_routing", "lora", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.1286266759998398, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 993.4664925979996, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/models (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))"}, "traceback": [{"path": "llmisvc/test_llm_inference_service.py", "lineno": 816, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1119, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError"}], "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebbf393d0>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebbf393d0>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n>           response = conn.getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:534: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebbf393d0>\n\n    def getresponse(  # type: ignore[override]\n        self,\n    ) -> HTTPResponse:\n        \"\"\"\n        Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.\n    \n        If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.\n        \"\"\"\n        # Raise the same error as http.client.HTTPConnection\n        if self._response_options is None:\n            raise ResponseNotReady()\n    \n        # Reset this attribute for being used again.\n        resp_options = self._response_options\n        self._response_options = None\n    \n        # Since the connection's timeout value may have been updated\n        # we need to set the timeout on the socket.\n        self.sock.settimeout(self.timeout)\n    \n        # This is needed here to avoid circular import errors\n        from .response import HTTPResponse\n    \n        # Save a reference to the shutdown function before ownership is passed\n        # to httplib_response\n        # TODO should we implement it everywhere?\n        _shutdown = getattr(self.sock, \"shutdown\", None)\n    \n        # Get the response from http.client.HTTPConnection\n>       httplib_response = super().getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:571: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebbf393d0>\n\n    def getresponse(self):\n        \"\"\"Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an\n        instance of HTTPResponse or of whatever object is returned by\n        the response_class variable.\n    \n        If a request has not been sent or if a previous response has\n        not be handled, ResponseNotReady is raised.  If the HTTP\n        response indicates that the connection should be closed, then\n        it will be closed before the response is returned.  When the\n        connection is closed, the underlying socket is closed.\n        \"\"\"\n    \n        # if a prior response has been completed, then forget about it.\n        if self.__response and self.__response.isclosed():\n            self.__response = None\n    \n        # if a prior response exists, then it must be completed (otherwise, we\n        # cannot read this response's header to determine the connection-close\n        # behavior)\n        #\n        # note: if a prior response existed, but was connection-close, then the\n        # socket and response were made independent of this HTTPConnection\n        # object since a new request requires that we open a whole new\n        # connection\n        #\n        # this means the prior response had one of two states:\n        #   1) will_close: this connection was reset and the prior socket and\n        #                  response operate independently\n        #   2) persistent: the response was retained and we await its\n        #                  isclosed() status to become true.\n        #\n        if self.__state != _CS_REQ_SENT or self.__response:\n            raise ResponseNotReady(self.__state)\n    \n        if self.debuglevel > 0:\n            response = self.response_class(self.sock, self.debuglevel,\n                                           method=self._method)\n        else:\n            response = self.response_class(self.sock, method=self._method)\n    \n        try:\n            try:\n>               response.begin()\n\n/usr/lib64/python3.11/http/client.py:1395: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7fcebc5f28c0>\n\n    def begin(self):\n        if self.headers is not None:\n            # we've already started reading the response\n            return\n    \n        # read until we get a non-100 response\n        while True:\n>           version, status, reason = self._read_status()\n\n/usr/lib64/python3.11/http/client.py:325: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7fcebc5f28c0>\n\n    def _read_status(self):\n>       line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n\n/usr/lib64/python3.11/http/client.py:286: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <socket.SocketIO object at 0x7fcebc5f1ba0>\nb = <memory at 0x7fcebc0c6b00>\n\n    def readinto(self, b):\n        \"\"\"Read up to len(b) bytes into the writable buffer *b* and return\n        the number of bytes read.  If the socket is non-blocking and no bytes\n        are available, None is returned.\n    \n        If *b* is non-empty, a 0 return value indicates that the connection\n        was shutdown at the other end.\n        \"\"\"\n        self._checkClosed()\n        self._checkReadable()\n        if self._timeout_occurred:\n            raise OSError(\"cannot read from timed out object\")\n        while True:\n            try:\n>               return self._sock.recv_into(b)\nE               TimeoutError: timed out\n\n/usr/lib64/python3.11/socket.py:718: TimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n>           response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:787: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebbf393d0>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7fcebbf393d0>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n            response = conn.getresponse()\n        except (BaseSSLError, OSError) as e:\n>           self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:536: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nerr = TimeoutError('timed out'), url = '/v1/models', timeout_value = 60\n\n    def _raise_timeout(\n        self,\n        err: BaseSSLError | OSError | SocketTimeout,\n        url: str,\n        timeout_value: _TYPE_TIMEOUT | None,\n    ) -> None:\n        \"\"\"Is the error actually a timeout? Will raise a ReadTimeout or pass\"\"\"\n    \n        if isinstance(err, SocketTimeout):\n>           raise ReadTimeoutError(\n                self, url, f\"Read timed out. (read timeout={timeout_value})\"\n            ) from err\nE           urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <requests.adapters.HTTPAdapter object at 0x7fcebbb6a8d0>\nrequest = <PreparedRequest [GET]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n>           resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:667: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=7, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=6, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=5, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=4, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=3, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=2, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=1, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\nmethod = 'GET', url = '/v1/models', body = None\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/v1/models', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n>           retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:841: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nmethod = 'GET', url = '/v1/models', response = None\nerror = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\n_pool = <urllib3.connectionpool.HTTPConnectionPool object at 0x7fcebbb69e90>\n_stacktrace = <traceback object at 0x7fcebbf3ab40>\n\n    def increment(\n        self,\n        method: str | None = None,\n        url: str | None = None,\n        response: BaseHTTPResponse | None = None,\n        error: Exception | None = None,\n        _pool: ConnectionPool | None = None,\n        _stacktrace: TracebackType | None = None,\n    ) -> Self:\n        \"\"\"Return a new Retry object with incremented retry counters.\n    \n        :param response: A response object, or None, if the server did not\n            return a response.\n        :type response: :class:`~urllib3.response.BaseHTTPResponse`\n        :param Exception error: An error encountered during the request, or\n            None if the response was received successfully.\n    \n        :return: A new ``Retry`` object.\n        \"\"\"\n        if self.total is False and error:\n            # Disabled, indicate to re-raise the error.\n            raise reraise(type(error), error, _stacktrace)\n    \n        total = self.total\n        if total is not None:\n            total -= 1\n    \n        connect = self.connect\n        read = self.read\n        redirect = self.redirect\n        status_count = self.status\n        other = self.other\n        cause = \"unknown\"\n        status = None\n        redirect_location = None\n    \n        if error and self._is_connection_error(error):\n            # Connect retry?\n            if connect is False:\n                raise reraise(type(error), error, _stacktrace)\n            elif connect is not None:\n                connect -= 1\n    \n        elif error and self._is_read_error(error):\n            # Read retry?\n            if read is False or method is None or not self._is_method_retryable(method):\n                raise reraise(type(error), error, _stacktrace)\n            elif read is not None:\n                read -= 1\n    \n        elif error:\n            # Other retry?\n            if other is not None:\n                other -= 1\n    \n        elif response and response.get_redirect_location():\n            # Redirect retry?\n            if redirect is not None:\n                redirect -= 1\n            cause = \"too many redirects\"\n            response_redirect_location = response.get_redirect_location()\n            if response_redirect_location:\n                redirect_location = response_redirect_location\n            status = response.status\n    \n        else:\n            # Incrementing because of a server error like a 500 in\n            # status_forcelist and the given method is in the allowed_methods\n            cause = ResponseError.GENERIC_ERROR\n            if response and response.status:\n                if status_count is not None:\n                    status_count -= 1\n                cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)\n                status = response.status\n    \n        history = self.history + (\n            RequestHistory(method, url, error, status, redirect_location),\n        )\n    \n        new_retry = self.new(\n            total=total,\n            connect=connect,\n            read=read,\n            redirect=redirect,\n            status=status_count,\n            other=other,\n            history=history,\n        )\n    \n        if new_retry.is_exhausted():\n            reason = error or ResponseError(cause)\n>           raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\nE           urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/models (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:519: MaxRetryError\n\nDuring handling of the above exception, another exception occurred:\n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n                response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n            else:\n>               response = get_with_retry(\n                    model_url,\n                    headers=headers,\n                    timeout=test_case.response_timeout,\n                )\n\nllmisvc/test_llm_inference_service.py:1102: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/v1/models'\n\n    def get_with_retry(\n        url: str,\n        *,\n        headers: Dict = None,\n        timeout: float = None,\n        total_retries: int = DEFAULT_RETRY_TOTAL,\n        backoff_factor: float = DEFAULT_RETRY_BACKOFF_FACTOR,\n        retry_status_codes=DEFAULT_RETRY_STATUS_CODES,\n    ) -> requests.Response:\n        \"\"\"\n        Send GET request with retries for transient HTTP and network failures.\n        \"\"\"\n        with _retry_session(\n            [\"GET\"], total_retries, backoff_factor, retry_status_codes\n        ) as session:\n>           return session.get(url, headers=headers, timeout=timeout)\n\ncommon/http_retry.py:46: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7fcebbfc40d0>\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/v1/models'\nkwargs = {'allow_redirects': True, 'headers': {'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}, 'timeout': 60}\n\n    def get(self, url, **kwargs):\n        r\"\"\"Sends a GET request. Returns :class:`Response` object.\n    \n        :param url: URL for the new :class:`Request` object.\n        :param \\*\\*kwargs: Optional arguments that ``request`` takes.\n        :rtype: requests.Response\n        \"\"\"\n    \n        kwargs.setdefault(\"allow_redirects\", True)\n>       return self.request(\"GET\", url, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:602: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7fcebbfc40d0>, method = 'GET'\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/v1/models'\nparams = None, data = None\nheaders = {'Content-Type': 'application/json', 'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\ncookies = None, files = None, auth = None, timeout = 60, allow_redirects = True\nproxies = {}, hooks = None, stream = None, verify = None, cert = None\njson = None\n\n    def request(\n        self,\n        method,\n        url,\n        params=None,\n        data=None,\n        headers=None,\n        cookies=None,\n        files=None,\n        auth=None,\n        timeout=None,\n        allow_redirects=True,\n        proxies=None,\n        hooks=None,\n        stream=None,\n        verify=None,\n        cert=None,\n        json=None,\n    ):\n        \"\"\"Constructs a :class:`Request <Request>`, prepares it and sends it.\n        Returns :class:`Response <Response>` object.\n    \n        :param method: method for the new :class:`Request` object.\n        :param url: URL for the new :class:`Request` object.\n        :param params: (optional) Dictionary or bytes to be sent in the query\n            string for the :class:`Request`.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the\n            :class:`Request`.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the\n            :class:`Request`.\n        :param cookies: (optional) Dict or CookieJar object to send with the\n            :class:`Request`.\n        :param files: (optional) Dictionary of ``'filename': file-like-objects``\n            for multipart encoding upload.\n        :param auth: (optional) Auth tuple or callable to enable\n            Basic/Digest/Custom HTTP Auth.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple\n        :param allow_redirects: (optional) Set to True by default.\n        :type allow_redirects: bool\n        :param proxies: (optional) Dictionary mapping protocol or protocol and\n            hostname to the URL of the proxy.\n        :param hooks: (optional) Dictionary mapping hook name to one event or\n            list of events, event must be callable.\n        :param stream: (optional) whether to immediately download the response\n            content. Defaults to ``False``.\n        :param verify: (optional) Either a boolean, in which case it controls whether we verify\n            the server's TLS certificate, or a string, in which case it must be a path\n            to a CA bundle to use. Defaults to ``True``. When set to\n            ``False``, requests will accept any TLS certificate presented by\n            the server, and will ignore hostname mismatches and/or expired\n            certificates, which will make your application vulnerable to\n            man-in-the-middle (MitM) attacks. Setting verify to ``False``\n            may be useful during local development or testing.\n        :param cert: (optional) if String, path to ssl client cert file (.pem).\n            If Tuple, ('cert', 'key') pair.\n        :rtype: requests.Response\n        \"\"\"\n        # Create the Request.\n        req = Request(\n            method=method.upper(),\n            url=url,\n            headers=headers,\n            files=files,\n            data=data or {},\n            json=json,\n            params=params or {},\n            auth=auth,\n            cookies=cookies,\n            hooks=hooks,\n        )\n        prep = self.prepare_request(req)\n    \n        proxies = proxies or {}\n    \n        settings = self.merge_environment_settings(\n            prep.url, proxies, stream, verify, cert\n        )\n    \n        # Send the request.\n        send_kwargs = {\n            \"timeout\": timeout,\n            \"allow_redirects\": allow_redirects,\n        }\n        send_kwargs.update(settings)\n>       resp = self.send(prep, **send_kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:589: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7fcebbfc40d0>\nrequest = <PreparedRequest [GET]>\nkwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': 60, ...}\nallow_redirects = True, stream = False, hooks = {'response': []}\nadapter = <requests.adapters.HTTPAdapter object at 0x7fcebbb6a8d0>\nstart = 1783426795.6312437\n\n    def send(self, request, **kwargs):\n        \"\"\"Send a given PreparedRequest.\n    \n        :rtype: requests.Response\n        \"\"\"\n        # Set defaults that the hooks can utilize to ensure they always have\n        # the correct parameters to reproduce the previous request.\n        kwargs.setdefault(\"stream\", self.stream)\n        kwargs.setdefault(\"verify\", self.verify)\n        kwargs.setdefault(\"cert\", self.cert)\n        if \"proxies\" not in kwargs:\n            kwargs[\"proxies\"] = resolve_proxies(request, self.proxies, self.trust_env)\n    \n        # It's possible that users might accidentally send a Request object.\n        # Guard against that specific failure case.\n        if isinstance(request, Request):\n            raise ValueError(\"You can only send PreparedRequests.\")\n    \n        # Set up variables needed for resolve_redirects and dispatching of hooks\n        allow_redirects = kwargs.pop(\"allow_redirects\", True)\n        stream = kwargs.get(\"stream\")\n        hooks = request.hooks\n    \n        # Get the appropriate adapter to use\n        adapter = self.get_adapter(url=request.url)\n    \n        # Start time (approximately) of the request\n        start = preferred_clock()\n    \n        # Send the request\n>       r = adapter.send(request, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:703: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.adapters.HTTPAdapter object at 0x7fcebbb6a8d0>\nrequest = <PreparedRequest [GET]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n            resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n    \n        except (ProtocolError, OSError) as err:\n            raise ConnectionError(err, request=request)\n    \n        except MaxRetryError as e:\n            if isinstance(e.reason, ConnectTimeoutError):\n                # TODO: Remove this in 3.0.0: see #2811\n                if not isinstance(e.reason, NewConnectionError):\n                    raise ConnectTimeout(e, request=request)\n    \n            if isinstance(e.reason, ResponseError):\n                raise RetryError(e, request=request)\n    \n            if isinstance(e.reason, _ProxyError):\n                raise ProxyError(e, request=request)\n    \n            if isinstance(e.reason, _SSLError):\n                # This branch is for urllib3 v1.22 and later.\n                raise SSLError(e, request=request)\n    \n>           raise ConnectionError(e, request=request)\nE           requests.exceptions.ConnectionError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/models (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:700: ConnectionError\n\nThe above exception was the direct cause of the following exception:\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m-with-lora-hf'], prompt=None, service_n...               {'name': 'model-fb-opt-125m-with-lora-hf-c0d503b0'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.asyncio(loop_scope=\"session\")\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-gateway-ref\",\n                        \"router-with-managed-route\",\n                        \"model-fb-opt-125m\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"custom-route-timeout-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"router-with-refs-test\",\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                            routes=[ROUTER_ROUTES[0], ROUTER_ROUTES[1]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\"router-managed\", \"workload-pd-cpu\", \"model-fb-opt-125m\"],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"custom-route-timeout-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"router-with-refs-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                    expected_gateway=ROUTER_GATEWAYS[1],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[1]],\n                            routes=[ROUTER_ROUTES[2], ROUTER_ROUTES[3]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-dp-ep-gpu\",\n                        \"workload-dp-ep-prefill-gpu\",\n                        \"model-deepseek-v2-lite\",\n                    ],\n                    prompt=\"Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically \"\n                    \"where the compute plane (P) and the data plane (D) are independently deployed and managed for a \"\n                    \"geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the \"\n                    \"fundamental challenges of network latency and data consistency, elaborate on the advanced \"\n                    \"considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: \"\n                    \"How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to \"\n                    \"evolve to support optimal performance and minimize inter-plane communication overhead, especially for \"\n                    \"synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically \"\n                    \"optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: \"\n                    \"Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) \"\n                    \"and their applicability in balancing performance and data integrity across a globally distributed data plane. \"\n                    \"Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, \"\n                    \"intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. \"\n                    \"3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently \"\n                    \"manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, \"\n                    \"cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). \"\n                    \"Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on \"\n                    \"workload patterns and data locality, potentially involving live migration strategies. \"\n                    \"4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter \"\n                    \"challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), \"\n                    \"fine-grained access control to data at rest and in motion, and identity management across disaggregated \"\n                    \"components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) \"\n                    \"concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: \"\n                    \"Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and \"\n                    \"data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) \"\n                    \"would be essential? How would incident response and troubleshooting differ in this disaggregated environment \"\n                    \"compared to traditional integrated systems? Consider the challenges of pinpointing root causes across \"\n                    \"independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries \"\n                    \"or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) \"\n                    \"where the benefits of P/D disaggregation would strongly outweigh its complexities. \"\n                    \"Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions \"\n                    \"directly interacting with object storage, in-memory disaggregation) that could further drive or \"\n                    \"transform P/D disaggregation in cloud computing.\",\n                    max_tokens=2000,\n                ),\n                marks=[\n                    pytest.mark.cluster_gpu,\n                    pytest.mark.cluster_nvidia,\n                    pytest.mark.cluster_nvidia_roce,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-no-scheduler\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"What is KServe?\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.no_scheduler,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, \"\n                    \"but without the resources requirements for DP+EP (GPUs and ROCe/IB).\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],\n            ),\n            # Scheduler config tests\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-inline-config\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-inline-config-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Chat completions endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                        \"model-qwen2.5-0.5b\",\n                    ],\n                    model_name=\"Qwen/Qwen2.5-0.5B-Instruct\",\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-configmap-ref\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-configmap-ref-test\",\n                    before_test=[create_scheduler_configmap],\n                    after_test=[delete_scheduler_configmap],\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-replicas\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-ha-replicas-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-custom-template\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-custom-template-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Scheduler v0.6 \u2192 v0.7 migration tests.\n            # Deploy v0.6-style configs and verify the controller migrates them\n            # so the v0.7 scheduler boots successfully.\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-pd-config-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-pd-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-nonzero-threshold-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-threshold-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Precise prefix KV cache routing test\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-precise-prefix-cache-inline-config\",\n                        \"workload-llmd-simulator-kvcache\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"precise-prefix-cache-test\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Models endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=create_response_assertion(with_field=\"data\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/completions\",\n                            prompt=\"KServe is a\",\n                            payload_formatter=completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/chat/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/chat/completions\",\n                            prompt=\"What is KServe?\",\n                            payload_formatter=chat_completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 LoRA adapter\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    model_name=f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\"\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/models (base + LoRA)\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=assert_models_contains(\n                        \"facebook/opt-125m\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                        \"lora-adapter-1\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # PVC storage tests -- validate direct PVC volume mount with real vLLM serving\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_multi_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_llm_inference_service(test_case: TestCase):  # noqa: F811\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        if not test_case.llm_service.metadata.annotations:\n            test_case.llm_service.metadata.annotations = {}\n    \n        test_case.llm_service.metadata.annotations[\n            \"security.opendatahub.io/enable-auth\"\n        ] = \"false\"\n        prefix = test_case.log_prefix\n    \n        test_failed = False\n        try:\n            print(f\"{prefix} Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n            print(f\"{prefix} Waiting for LLMInferenceService {service_name} to be ready\")\n            wait_for_llm_isvc_ready(\n                kserve_client, test_case.llm_service, test_case.wait_timeout\n            )\n            print(f\"{prefix} Waiting for model response from {service_name}\")\n>           wait_for_model_response(\n                kserve_client,\n                test_case,\n                test_case.wait_timeout,\n                extra_headers=test_case.extra_headers,\n            )\n\nllmisvc/test_llm_inference_service.py:816: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (<kserve.api.kserve_client.KServeClient object at 0x7fcebbba6e10>, TestCase(base_refs=['router-managed', 'workload-sin...         {'name': 'model-fb-opt-125m-with-lora-hf-c0d503b0'}]},\n 'status': None}, model_name='facebook/opt-125m'), 900)\nkwargs = {'extra_headers': {'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}}\nfunc_name = 'wait_for_model_response'\ntimestamp_start = '2026-07-07T12:19:55.472590', start_time = 1783426795.47287\nduration = 904.5829637050629, timestamp_end = '2026-07-07T12:35:00.055837'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fcebbba6e10>\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m-with-lora-hf'], prompt=None, service_n...               {'name': 'model-fb-opt-125m-with-lora-hf-c0d503b0'}]},\n 'status': None}, model_name='facebook/opt-125m')\ntimeout_seconds = 900\nextra_headers = {'X-Gateway-Model-Name': 'publishers/kserve-ci-e2e-test/models/facebook/opt-125m'}\n\n    @log_execution\n    def wait_for_model_response(\n        kserve_client: KServeClient,\n        test_case: TestCase,  # noqa: F811\n        timeout_seconds: int = 900,\n        extra_headers: Optional[Dict[str, str]] = None,\n    ) -> str:\n        def get_successful_response():\n            try:\n                if test_case.url_getter:\n                    service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n                else:\n                    service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n            except Exception as e:\n                raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n            model_url = service_url + test_case.endpoint\n    \n            headers = {\"Content-Type\": \"application/json\"}\n            if extra_headers:\n                headers.update(extra_headers)\n    \n            if test_case.payload_formatter is not None:\n                test_payload = test_case.payload_formatter(test_case)\n            elif test_case.prompt is not None:\n                test_payload = {\n                    \"model\": test_case.model_name\n                    if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                    else extra_headers[MODEL_ROUTING_HEADER],\n                    \"prompt\": test_case.prompt,\n                    \"max_tokens\": test_case.max_tokens,\n                }\n            else:\n                test_payload = None\n    \n            logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n            try:\n                if test_payload is not None:\n                    response = post_with_retry(\n                        model_url,\n                        headers=headers,\n                        json_data=test_payload,\n                        timeout=test_case.response_timeout,\n                    )\n                else:\n                    response = get_with_retry(\n                        model_url,\n                        headers=headers,\n                        timeout=test_case.response_timeout,\n                    )\n            except Exception as e:\n                logger.error(f\"\u274c Failed to call model: {e}\")\n                raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\n    \n            logger.info(f\"Model response is {response.status_code}: {response.text[:500]}\")\n    \n            if 200 <= response.status_code < 300:\n                return response\n            raise AssertionError(\n                f\"Service returned {response.status_code}: {response.text}\"\n            )\n    \n>       response = wait_for(get_successful_response, timeout=timeout_seconds, interval=5.0)\n\nllmisvc/test_llm_inference_service.py:1119: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_model_response.<locals>.get_successful_response at 0x7fcebc196d40>\ntimeout = 900, interval = 5.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n                response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n            else:\n                response = get_with_retry(\n                    model_url,\n                    headers=headers,\n                    timeout=test_case.response_timeout,\n                )\n        except Exception as e:\n            logger.error(f\"\u274c Failed to call model: {e}\")\n>           raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\nE           AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /v1/models (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\nllmisvc/test_llm_inference_service.py:1109: AssertionError"}, "teardown": {"duration": 0.0011383040000509936, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-custom-route-timeout-scheduler-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "failed", "keywords": ["test_llm_inference_service[router-custom-route-timeout-scheduler-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-custom-route-timeout-scheduler-managed-workload-single-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.18020540599900414, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 1034.9016673289989, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/custom-route-timeout-test/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))"}, "traceback": [{"path": "llmisvc/test_llm_inference_service.py", "lineno": 816, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1119, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1109, "message": "AssertionError"}], "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953f11cf50>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953f11cf50>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n>           response = conn.getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:534: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953f11cf50>\n\n    def getresponse(  # type: ignore[override]\n        self,\n    ) -> HTTPResponse:\n        \"\"\"\n        Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an instance of HTTPResponse or of whatever object is returned by the response_class variable.\n    \n        If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP response indicates that the connection should be closed, then it will be closed before the response is returned. When the connection is closed, the underlying socket is closed.\n        \"\"\"\n        # Raise the same error as http.client.HTTPConnection\n        if self._response_options is None:\n            raise ResponseNotReady()\n    \n        # Reset this attribute for being used again.\n        resp_options = self._response_options\n        self._response_options = None\n    \n        # Since the connection's timeout value may have been updated\n        # we need to set the timeout on the socket.\n        self.sock.settimeout(self.timeout)\n    \n        # This is needed here to avoid circular import errors\n        from .response import HTTPResponse\n    \n        # Save a reference to the shutdown function before ownership is passed\n        # to httplib_response\n        # TODO should we implement it everywhere?\n        _shutdown = getattr(self.sock, \"shutdown\", None)\n    \n        # Get the response from http.client.HTTPConnection\n>       httplib_response = super().getresponse()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:571: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953f11cf50>\n\n    def getresponse(self):\n        \"\"\"Get the response from the server.\n    \n        If the HTTPConnection is in the correct state, returns an\n        instance of HTTPResponse or of whatever object is returned by\n        the response_class variable.\n    \n        If a request has not been sent or if a previous response has\n        not be handled, ResponseNotReady is raised.  If the HTTP\n        response indicates that the connection should be closed, then\n        it will be closed before the response is returned.  When the\n        connection is closed, the underlying socket is closed.\n        \"\"\"\n    \n        # if a prior response has been completed, then forget about it.\n        if self.__response and self.__response.isclosed():\n            self.__response = None\n    \n        # if a prior response exists, then it must be completed (otherwise, we\n        # cannot read this response's header to determine the connection-close\n        # behavior)\n        #\n        # note: if a prior response existed, but was connection-close, then the\n        # socket and response were made independent of this HTTPConnection\n        # object since a new request requires that we open a whole new\n        # connection\n        #\n        # this means the prior response had one of two states:\n        #   1) will_close: this connection was reset and the prior socket and\n        #                  response operate independently\n        #   2) persistent: the response was retained and we await its\n        #                  isclosed() status to become true.\n        #\n        if self.__state != _CS_REQ_SENT or self.__response:\n            raise ResponseNotReady(self.__state)\n    \n        if self.debuglevel > 0:\n            response = self.response_class(self.sock, self.debuglevel,\n                                           method=self._method)\n        else:\n            response = self.response_class(self.sock, method=self._method)\n    \n        try:\n            try:\n>               response.begin()\n\n/usr/lib64/python3.11/http/client.py:1395: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7f95409dd960>\n\n    def begin(self):\n        if self.headers is not None:\n            # we've already started reading the response\n            return\n    \n        # read until we get a non-100 response\n        while True:\n>           version, status, reason = self._read_status()\n\n/usr/lib64/python3.11/http/client.py:325: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <http.client.HTTPResponse object at 0x7f95409dd960>\n\n    def _read_status(self):\n>       line = str(self.fp.readline(_MAXLINE + 1), \"iso-8859-1\")\n\n/usr/lib64/python3.11/http/client.py:286: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <socket.SocketIO object at 0x7f95409dfe80>\nb = <memory at 0x7f953fa02d40>\n\n    def readinto(self, b):\n        \"\"\"Read up to len(b) bytes into the writable buffer *b* and return\n        the number of bytes read.  If the socket is non-blocking and no bytes\n        are available, None is returned.\n    \n        If *b* is non-empty, a 0 return value indicates that the connection\n        was shutdown at the other end.\n        \"\"\"\n        self._checkClosed()\n        self._checkReadable()\n        if self._timeout_occurred:\n            raise OSError(\"cannot read from timed out object\")\n        while True:\n            try:\n>               return self._sock.recv_into(b)\nE               TimeoutError: timed out\n\n/usr/lib64/python3.11/socket.py:718: TimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n>           response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:787: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nconn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953f11cf50>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=60, read=60, total=None), chunked = False\nresponse_conn = <HTTPConnection(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80) at 0x7f953f11cf50>\npreload_content = False, decode_content = False, enforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            raise new_e\n    \n        # conn.request() calls http.client.*.request, not the method in\n        # urllib3.request. It also calls makefile (recv) on the socket.\n        try:\n            conn.request(\n                method,\n                url,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                enforce_content_length=enforce_content_length,\n            )\n    \n        # We are swallowing BrokenPipeError (errno.EPIPE) since the server is\n        # legitimately able to close the connection after sending a valid response.\n        # With this behaviour, the received response is still readable.\n        except BrokenPipeError:\n            pass\n        except OSError as e:\n            # MacOS/Linux\n            # EPROTOTYPE and ECONNRESET are needed on macOS\n            # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/\n            # Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.\n            if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:\n                raise\n    \n        # Reset the timeout for the recv() on the socket\n        read_timeout = timeout_obj.read_timeout\n    \n        if not conn.is_closed:\n            # In Python 3 socket.py will catch EAGAIN and return None when you\n            # try and read into the file pointer created by http.client, which\n            # instead raises a BadStatusLine exception. Instead of catching\n            # the exception and assuming all BadStatusLine exceptions are read\n            # timeouts, check for a zero timeout before making the request.\n            if read_timeout == 0:\n                raise ReadTimeoutError(\n                    self, url, f\"Read timed out. (read timeout={read_timeout})\"\n                )\n            conn.timeout = read_timeout\n    \n        # Receive the response from the server\n        try:\n            response = conn.getresponse()\n        except (BaseSSLError, OSError) as e:\n>           self._raise_timeout(err=e, url=url, timeout_value=read_timeout)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:536: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nerr = TimeoutError('timed out')\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\ntimeout_value = 60\n\n    def _raise_timeout(\n        self,\n        err: BaseSSLError | OSError | SocketTimeout,\n        url: str,\n        timeout_value: _TYPE_TIMEOUT | None,\n    ) -> None:\n        \"\"\"Is the error actually a timeout? Will raise a ReadTimeout or pass\"\"\"\n    \n        if isinstance(err, SocketTimeout):\n>           raise ReadTimeoutError(\n                self, url, f\"Read timed out. (read timeout={timeout_value})\"\n            ) from err\nE           urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError\n\nThe above exception was the direct cause of the following exception:\n\nself = <requests.adapters.HTTPAdapter object at 0x7f953f948490>\nrequest = <PreparedRequest [POST]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n>           resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:667: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=7, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=6, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=5, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=4, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=3, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=2, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=1, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nbody = b'{\"model\": \"facebook/opt-125m\", \"prompt\": \"KServe is a\", \"max_tokens\": 20}'\nheaders = {'User-Agent': 'python-requests/2.32.3', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Content-Length': '73'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False\ntimeout = Timeout(connect=60, read=60, total=None), pool_timeout = None\nrelease_conn = False, chunked = False, body_pos = None, preload_content = False\ndecode_content = False, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions', query=None, fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n>           retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:841: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nmethod = 'POST'\nurl = '/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nresponse = None\nerror = ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\")\n_pool = <urllib3.connectionpool.HTTPConnectionPool object at 0x7f953fa8a1d0>\n_stacktrace = <traceback object at 0x7f953f11ee00>\n\n    def increment(\n        self,\n        method: str | None = None,\n        url: str | None = None,\n        response: BaseHTTPResponse | None = None,\n        error: Exception | None = None,\n        _pool: ConnectionPool | None = None,\n        _stacktrace: TracebackType | None = None,\n    ) -> Self:\n        \"\"\"Return a new Retry object with incremented retry counters.\n    \n        :param response: A response object, or None, if the server did not\n            return a response.\n        :type response: :class:`~urllib3.response.BaseHTTPResponse`\n        :param Exception error: An error encountered during the request, or\n            None if the response was received successfully.\n    \n        :return: A new ``Retry`` object.\n        \"\"\"\n        if self.total is False and error:\n            # Disabled, indicate to re-raise the error.\n            raise reraise(type(error), error, _stacktrace)\n    \n        total = self.total\n        if total is not None:\n            total -= 1\n    \n        connect = self.connect\n        read = self.read\n        redirect = self.redirect\n        status_count = self.status\n        other = self.other\n        cause = \"unknown\"\n        status = None\n        redirect_location = None\n    \n        if error and self._is_connection_error(error):\n            # Connect retry?\n            if connect is False:\n                raise reraise(type(error), error, _stacktrace)\n            elif connect is not None:\n                connect -= 1\n    \n        elif error and self._is_read_error(error):\n            # Read retry?\n            if read is False or method is None or not self._is_method_retryable(method):\n                raise reraise(type(error), error, _stacktrace)\n            elif read is not None:\n                read -= 1\n    \n        elif error:\n            # Other retry?\n            if other is not None:\n                other -= 1\n    \n        elif response and response.get_redirect_location():\n            # Redirect retry?\n            if redirect is not None:\n                redirect -= 1\n            cause = \"too many redirects\"\n            response_redirect_location = response.get_redirect_location()\n            if response_redirect_location:\n                redirect_location = response_redirect_location\n            status = response.status\n    \n        else:\n            # Incrementing because of a server error like a 500 in\n            # status_forcelist and the given method is in the allowed_methods\n            cause = ResponseError.GENERIC_ERROR\n            if response and response.status:\n                if status_count is not None:\n                    status_count -= 1\n                cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)\n                status = response.status\n    \n        history = self.history + (\n            RequestHistory(method, url, error, status, redirect_location),\n        )\n    \n        new_retry = self.new(\n            total=total,\n            connect=connect,\n            read=read,\n            redirect=redirect,\n            status=status_count,\n            other=other,\n            history=history,\n        )\n    \n        if new_retry.is_exhausted():\n            reason = error or ResponseError(cause)\n>           raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\nE           urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/custom-route-timeout-test/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:519: MaxRetryError\n\nDuring handling of the above exception, another exception occurred:\n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n>               response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n\nllmisvc/test_llm_inference_service.py:1095: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\n\n    def post_with_retry(\n        url: str,\n        *,\n        headers: Dict = None,\n        json_data: Union[Dict, List] = None,\n        data: Union[str, bytes] = None,\n        stream: bool = False,\n        timeout: float = None,\n        total_retries: int = DEFAULT_RETRY_TOTAL,\n        backoff_factor: float = DEFAULT_RETRY_BACKOFF_FACTOR,\n        retry_status_codes=DEFAULT_RETRY_STATUS_CODES,\n    ) -> requests.Response:\n        \"\"\"\n        Send POST request with retries for transient HTTP and network failures.\n        \"\"\"\n        if json_data is not None and data is not None:\n            raise ValueError(\"Only one of json_data or data can be provided.\")\n    \n        with _retry_session(\n            [\"POST\"], total_retries, backoff_factor, retry_status_codes\n        ) as session:\n>           return session.post(\n                url,\n                json=json_data,\n                data=data,\n                headers=headers,\n                stream=stream,\n                timeout=timeout,\n            )\n\ncommon/http_retry.py:70: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7f953f903990>\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\ndata = None\njson = {'max_tokens': 20, 'model': 'facebook/opt-125m', 'prompt': 'KServe is a'}\nkwargs = {'headers': {'Content-Type': 'application/json'}, 'stream': False, 'timeout': 60}\n\n    def post(self, url, data=None, json=None, **kwargs):\n        r\"\"\"Sends a POST request. Returns :class:`Response` object.\n    \n        :param url: URL for the new :class:`Request` object.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the :class:`Request`.\n        :param \\*\\*kwargs: Optional arguments that ``request`` takes.\n        :rtype: requests.Response\n        \"\"\"\n    \n>       return self.request(\"POST\", url, data=data, json=json, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:637: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7f953f903990>, method = 'POST'\nurl = 'http://afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com/kserve-ci-e2e-test/custom-route-timeout-test/v1/completions'\nparams = None, data = None, headers = {'Content-Type': 'application/json'}\ncookies = None, files = None, auth = None, timeout = 60, allow_redirects = True\nproxies = {}, hooks = None, stream = False, verify = None, cert = None\njson = {'max_tokens': 20, 'model': 'facebook/opt-125m', 'prompt': 'KServe is a'}\n\n    def request(\n        self,\n        method,\n        url,\n        params=None,\n        data=None,\n        headers=None,\n        cookies=None,\n        files=None,\n        auth=None,\n        timeout=None,\n        allow_redirects=True,\n        proxies=None,\n        hooks=None,\n        stream=None,\n        verify=None,\n        cert=None,\n        json=None,\n    ):\n        \"\"\"Constructs a :class:`Request <Request>`, prepares it and sends it.\n        Returns :class:`Response <Response>` object.\n    \n        :param method: method for the new :class:`Request` object.\n        :param url: URL for the new :class:`Request` object.\n        :param params: (optional) Dictionary or bytes to be sent in the query\n            string for the :class:`Request`.\n        :param data: (optional) Dictionary, list of tuples, bytes, or file-like\n            object to send in the body of the :class:`Request`.\n        :param json: (optional) json to send in the body of the\n            :class:`Request`.\n        :param headers: (optional) Dictionary of HTTP Headers to send with the\n            :class:`Request`.\n        :param cookies: (optional) Dict or CookieJar object to send with the\n            :class:`Request`.\n        :param files: (optional) Dictionary of ``'filename': file-like-objects``\n            for multipart encoding upload.\n        :param auth: (optional) Auth tuple or callable to enable\n            Basic/Digest/Custom HTTP Auth.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple\n        :param allow_redirects: (optional) Set to True by default.\n        :type allow_redirects: bool\n        :param proxies: (optional) Dictionary mapping protocol or protocol and\n            hostname to the URL of the proxy.\n        :param hooks: (optional) Dictionary mapping hook name to one event or\n            list of events, event must be callable.\n        :param stream: (optional) whether to immediately download the response\n            content. Defaults to ``False``.\n        :param verify: (optional) Either a boolean, in which case it controls whether we verify\n            the server's TLS certificate, or a string, in which case it must be a path\n            to a CA bundle to use. Defaults to ``True``. When set to\n            ``False``, requests will accept any TLS certificate presented by\n            the server, and will ignore hostname mismatches and/or expired\n            certificates, which will make your application vulnerable to\n            man-in-the-middle (MitM) attacks. Setting verify to ``False``\n            may be useful during local development or testing.\n        :param cert: (optional) if String, path to ssl client cert file (.pem).\n            If Tuple, ('cert', 'key') pair.\n        :rtype: requests.Response\n        \"\"\"\n        # Create the Request.\n        req = Request(\n            method=method.upper(),\n            url=url,\n            headers=headers,\n            files=files,\n            data=data or {},\n            json=json,\n            params=params or {},\n            auth=auth,\n            cookies=cookies,\n            hooks=hooks,\n        )\n        prep = self.prepare_request(req)\n    \n        proxies = proxies or {}\n    \n        settings = self.merge_environment_settings(\n            prep.url, proxies, stream, verify, cert\n        )\n    \n        # Send the request.\n        send_kwargs = {\n            \"timeout\": timeout,\n            \"allow_redirects\": allow_redirects,\n        }\n        send_kwargs.update(settings)\n>       resp = self.send(prep, **send_kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:589: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.sessions.Session object at 0x7f953f903990>\nrequest = <PreparedRequest [POST]>\nkwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': 60, ...}\nallow_redirects = True, stream = False, hooks = {'response': []}\nadapter = <requests.adapters.HTTPAdapter object at 0x7f953f948490>\nstart = 1783426971.252987\n\n    def send(self, request, **kwargs):\n        \"\"\"Send a given PreparedRequest.\n    \n        :rtype: requests.Response\n        \"\"\"\n        # Set defaults that the hooks can utilize to ensure they always have\n        # the correct parameters to reproduce the previous request.\n        kwargs.setdefault(\"stream\", self.stream)\n        kwargs.setdefault(\"verify\", self.verify)\n        kwargs.setdefault(\"cert\", self.cert)\n        if \"proxies\" not in kwargs:\n            kwargs[\"proxies\"] = resolve_proxies(request, self.proxies, self.trust_env)\n    \n        # It's possible that users might accidentally send a Request object.\n        # Guard against that specific failure case.\n        if isinstance(request, Request):\n            raise ValueError(\"You can only send PreparedRequests.\")\n    \n        # Set up variables needed for resolve_redirects and dispatching of hooks\n        allow_redirects = kwargs.pop(\"allow_redirects\", True)\n        stream = kwargs.get(\"stream\")\n        hooks = request.hooks\n    \n        # Get the appropriate adapter to use\n        adapter = self.get_adapter(url=request.url)\n    \n        # Start time (approximately) of the request\n        start = preferred_clock()\n    \n        # Send the request\n>       r = adapter.send(request, **kwargs)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/sessions.py:703: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <requests.adapters.HTTPAdapter object at 0x7f953f948490>\nrequest = <PreparedRequest [POST]>, stream = False\ntimeout = Timeout(connect=60, read=60, total=None), verify = '/tmp/ca.crt'\ncert = None, proxies = OrderedDict()\n\n    def send(\n        self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None\n    ):\n        \"\"\"Sends PreparedRequest object. Returns Response object.\n    \n        :param request: The :class:`PreparedRequest <PreparedRequest>` being sent.\n        :param stream: (optional) Whether to stream the request content.\n        :param timeout: (optional) How long to wait for the server to send\n            data before giving up, as a float, or a :ref:`(connect timeout,\n            read timeout) <timeouts>` tuple.\n        :type timeout: float or tuple or urllib3 Timeout object\n        :param verify: (optional) Either a boolean, in which case it controls whether\n            we verify the server's TLS certificate, or a string, in which case it\n            must be a path to a CA bundle to use\n        :param cert: (optional) Any user-provided SSL certificate to be trusted.\n        :param proxies: (optional) The proxies dictionary to apply to the request.\n        :rtype: requests.Response\n        \"\"\"\n    \n        try:\n            conn = self.get_connection_with_tls_context(\n                request, verify, proxies=proxies, cert=cert\n            )\n        except LocationValueError as e:\n            raise InvalidURL(e, request=request)\n    \n        self.cert_verify(conn, request.url, verify, cert)\n        url = self.request_url(request, proxies)\n        self.add_headers(\n            request,\n            stream=stream,\n            timeout=timeout,\n            verify=verify,\n            cert=cert,\n            proxies=proxies,\n        )\n    \n        chunked = not (request.body is None or \"Content-Length\" in request.headers)\n    \n        if isinstance(timeout, tuple):\n            try:\n                connect, read = timeout\n                timeout = TimeoutSauce(connect=connect, read=read)\n            except ValueError:\n                raise ValueError(\n                    f\"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, \"\n                    f\"or a single float to set both timeouts to the same value.\"\n                )\n        elif isinstance(timeout, TimeoutSauce):\n            pass\n        else:\n            timeout = TimeoutSauce(connect=timeout, read=timeout)\n    \n        try:\n            resp = conn.urlopen(\n                method=request.method,\n                url=url,\n                body=request.body,\n                headers=request.headers,\n                redirect=False,\n                assert_same_host=False,\n                preload_content=False,\n                decode_content=False,\n                retries=self.max_retries,\n                timeout=timeout,\n                chunked=chunked,\n            )\n    \n        except (ProtocolError, OSError) as err:\n            raise ConnectionError(err, request=request)\n    \n        except MaxRetryError as e:\n            if isinstance(e.reason, ConnectTimeoutError):\n                # TODO: Remove this in 3.0.0: see #2811\n                if not isinstance(e.reason, NewConnectionError):\n                    raise ConnectTimeout(e, request=request)\n    \n            if isinstance(e.reason, ResponseError):\n                raise RetryError(e, request=request)\n    \n            if isinstance(e.reason, _ProxyError):\n                raise ProxyError(e, request=request)\n    \n            if isinstance(e.reason, _SSLError):\n                # This branch is for urllib3 v1.22 and later.\n                raise SSLError(e, request=request)\n    \n>           raise ConnectionError(e, request=request)\nE           requests.exceptions.ConnectionError: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/custom-route-timeout-test/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/requests/adapters.py:700: ConnectionError\n\nThe above exception was the direct cause of the following exception:\n\ntest_case = TestCase(base_refs=['router-custom-route-timeout', 'scheduler-managed', 'workload-single-cpu', 'model-fb-opt-125m'], p...               {'name': 'model-fb-opt-125m-custom-route-928a8601'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.asyncio(loop_scope=\"session\")\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-gateway-ref\",\n                        \"router-with-managed-route\",\n                        \"model-fb-opt-125m\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"custom-route-timeout-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"router-with-refs-test\",\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                            routes=[ROUTER_ROUTES[0], ROUTER_ROUTES[1]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\"router-managed\", \"workload-pd-cpu\", \"model-fb-opt-125m\"],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"custom-route-timeout-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"router-with-refs-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                    expected_gateway=ROUTER_GATEWAYS[1],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[1]],\n                            routes=[ROUTER_ROUTES[2], ROUTER_ROUTES[3]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-dp-ep-gpu\",\n                        \"workload-dp-ep-prefill-gpu\",\n                        \"model-deepseek-v2-lite\",\n                    ],\n                    prompt=\"Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically \"\n                    \"where the compute plane (P) and the data plane (D) are independently deployed and managed for a \"\n                    \"geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the \"\n                    \"fundamental challenges of network latency and data consistency, elaborate on the advanced \"\n                    \"considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: \"\n                    \"How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to \"\n                    \"evolve to support optimal performance and minimize inter-plane communication overhead, especially for \"\n                    \"synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically \"\n                    \"optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: \"\n                    \"Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) \"\n                    \"and their applicability in balancing performance and data integrity across a globally distributed data plane. \"\n                    \"Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, \"\n                    \"intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. \"\n                    \"3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently \"\n                    \"manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, \"\n                    \"cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). \"\n                    \"Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on \"\n                    \"workload patterns and data locality, potentially involving live migration strategies. \"\n                    \"4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter \"\n                    \"challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), \"\n                    \"fine-grained access control to data at rest and in motion, and identity management across disaggregated \"\n                    \"components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) \"\n                    \"concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: \"\n                    \"Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and \"\n                    \"data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) \"\n                    \"would be essential? How would incident response and troubleshooting differ in this disaggregated environment \"\n                    \"compared to traditional integrated systems? Consider the challenges of pinpointing root causes across \"\n                    \"independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries \"\n                    \"or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) \"\n                    \"where the benefits of P/D disaggregation would strongly outweigh its complexities. \"\n                    \"Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions \"\n                    \"directly interacting with object storage, in-memory disaggregation) that could further drive or \"\n                    \"transform P/D disaggregation in cloud computing.\",\n                    max_tokens=2000,\n                ),\n                marks=[\n                    pytest.mark.cluster_gpu,\n                    pytest.mark.cluster_nvidia,\n                    pytest.mark.cluster_nvidia_roce,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-no-scheduler\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"What is KServe?\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.no_scheduler,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, \"\n                    \"but without the resources requirements for DP+EP (GPUs and ROCe/IB).\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],\n            ),\n            # Scheduler config tests\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-inline-config\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-inline-config-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Chat completions endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                        \"model-qwen2.5-0.5b\",\n                    ],\n                    model_name=\"Qwen/Qwen2.5-0.5B-Instruct\",\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-configmap-ref\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-configmap-ref-test\",\n                    before_test=[create_scheduler_configmap],\n                    after_test=[delete_scheduler_configmap],\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-replicas\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-ha-replicas-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-custom-template\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-custom-template-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Scheduler v0.6 \u2192 v0.7 migration tests.\n            # Deploy v0.6-style configs and verify the controller migrates them\n            # so the v0.7 scheduler boots successfully.\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-pd-config-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-pd-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-nonzero-threshold-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-threshold-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Precise prefix KV cache routing test\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-precise-prefix-cache-inline-config\",\n                        \"workload-llmd-simulator-kvcache\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"precise-prefix-cache-test\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Models endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=create_response_assertion(with_field=\"data\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/completions\",\n                            prompt=\"KServe is a\",\n                            payload_formatter=completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/chat/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/chat/completions\",\n                            prompt=\"What is KServe?\",\n                            payload_formatter=chat_completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 LoRA adapter\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    model_name=f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\"\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/models (base + LoRA)\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=assert_models_contains(\n                        \"facebook/opt-125m\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                        \"lora-adapter-1\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # PVC storage tests -- validate direct PVC volume mount with real vLLM serving\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_multi_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_llm_inference_service(test_case: TestCase):  # noqa: F811\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        if not test_case.llm_service.metadata.annotations:\n            test_case.llm_service.metadata.annotations = {}\n    \n        test_case.llm_service.metadata.annotations[\n            \"security.opendatahub.io/enable-auth\"\n        ] = \"false\"\n        prefix = test_case.log_prefix\n    \n        test_failed = False\n        try:\n            print(f\"{prefix} Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n            print(f\"{prefix} Waiting for LLMInferenceService {service_name} to be ready\")\n            wait_for_llm_isvc_ready(\n                kserve_client, test_case.llm_service, test_case.wait_timeout\n            )\n            print(f\"{prefix} Waiting for model response from {service_name}\")\n>           wait_for_model_response(\n                kserve_client,\n                test_case,\n                test_case.wait_timeout,\n                extra_headers=test_case.extra_headers,\n            )\n\nllmisvc/test_llm_inference_service.py:816: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (<kserve.api.kserve_client.KServeClient object at 0x7f953f9ac6d0>, TestCase(base_refs=['router-custom-route-timeout', ...         {'name': 'model-fb-opt-125m-custom-route-928a8601'}]},\n 'status': None}, model_name='facebook/opt-125m'), 900)\nkwargs = {'extra_headers': None}, func_name = 'wait_for_model_response'\ntimestamp_start = '2026-07-07T12:22:51.240748', start_time = 1783426971.2416573\nduration = 904.5172309875488, timestamp_end = '2026-07-07T12:37:55.758891'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7f953f9ac6d0>\ntest_case = TestCase(base_refs=['router-custom-route-timeout', 'scheduler-managed', 'workload-single-cpu', 'model-fb-opt-125m'], p...               {'name': 'model-fb-opt-125m-custom-route-928a8601'}]},\n 'status': None}, model_name='facebook/opt-125m')\ntimeout_seconds = 900, extra_headers = None\n\n    @log_execution\n    def wait_for_model_response(\n        kserve_client: KServeClient,\n        test_case: TestCase,  # noqa: F811\n        timeout_seconds: int = 900,\n        extra_headers: Optional[Dict[str, str]] = None,\n    ) -> str:\n        def get_successful_response():\n            try:\n                if test_case.url_getter:\n                    service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n                else:\n                    service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n            except Exception as e:\n                raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n            model_url = service_url + test_case.endpoint\n    \n            headers = {\"Content-Type\": \"application/json\"}\n            if extra_headers:\n                headers.update(extra_headers)\n    \n            if test_case.payload_formatter is not None:\n                test_payload = test_case.payload_formatter(test_case)\n            elif test_case.prompt is not None:\n                test_payload = {\n                    \"model\": test_case.model_name\n                    if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                    else extra_headers[MODEL_ROUTING_HEADER],\n                    \"prompt\": test_case.prompt,\n                    \"max_tokens\": test_case.max_tokens,\n                }\n            else:\n                test_payload = None\n    \n            logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n            try:\n                if test_payload is not None:\n                    response = post_with_retry(\n                        model_url,\n                        headers=headers,\n                        json_data=test_payload,\n                        timeout=test_case.response_timeout,\n                    )\n                else:\n                    response = get_with_retry(\n                        model_url,\n                        headers=headers,\n                        timeout=test_case.response_timeout,\n                    )\n            except Exception as e:\n                logger.error(f\"\u274c Failed to call model: {e}\")\n                raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\n    \n            logger.info(f\"Model response is {response.status_code}: {response.text[:500]}\")\n    \n            if 200 <= response.status_code < 300:\n                return response\n            raise AssertionError(\n                f\"Service returned {response.status_code}: {response.text}\"\n            )\n    \n>       response = wait_for(get_successful_response, timeout=timeout_seconds, interval=5.0)\n\nllmisvc/test_llm_inference_service.py:1119: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_model_response.<locals>.get_successful_response at 0x7f953f94c900>\ntimeout = 900, interval = 5.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def get_successful_response():\n        try:\n            if test_case.url_getter:\n                service_url = test_case.url_getter(kserve_client, test_case.llm_service)\n            else:\n                service_url = get_llm_service_url(kserve_client, test_case.llm_service)\n        except Exception as e:\n            raise AssertionError(f\"\u274c Failed to get service URL: {e}\") from e\n    \n        model_url = service_url + test_case.endpoint\n    \n        headers = {\"Content-Type\": \"application/json\"}\n        if extra_headers:\n            headers.update(extra_headers)\n    \n        if test_case.payload_formatter is not None:\n            test_payload = test_case.payload_formatter(test_case)\n        elif test_case.prompt is not None:\n            test_payload = {\n                \"model\": test_case.model_name\n                if not extra_headers or MODEL_ROUTING_HEADER not in extra_headers\n                else extra_headers[MODEL_ROUTING_HEADER],\n                \"prompt\": test_case.prompt,\n                \"max_tokens\": test_case.max_tokens,\n            }\n        else:\n            test_payload = None\n    \n        logger.info(f\"Calling LLM service at {model_url} with payload {test_payload}\")\n        try:\n            if test_payload is not None:\n                response = post_with_retry(\n                    model_url,\n                    headers=headers,\n                    json_data=test_payload,\n                    timeout=test_case.response_timeout,\n                )\n            else:\n                response = get_with_retry(\n                    model_url,\n                    headers=headers,\n                    timeout=test_case.response_timeout,\n                )\n        except Exception as e:\n            logger.error(f\"\u274c Failed to call model: {e}\")\n>           raise AssertionError(f\"\u274c Failed to call model: {e}\") from e\nE           AssertionError: \u274c Failed to call model: HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Max retries exceeded with url: /kserve-ci-e2e-test/custom-route-timeout-test/v1/completions (Caused by ReadTimeoutError(\"HTTPConnectionPool(host='afb04aa38013643218e3c7e599ab0a98-359103574.us-east-1.elb.amazonaws.com', port=80): Read timed out. (read timeout=60)\"))\n\nllmisvc/test_llm_inference_service.py:1109: AssertionError"}, "teardown": {"duration": 0.0013514999991457444, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-pvc]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-single-cpu-model-pvc]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "pvc_storage", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-pvc", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 20.327547654000227, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 238.28089889699913, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.005800482998893131, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "custom_gateway", "__wrapped__", "pytestmark", "router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 2.160041863000515, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 191.23868119100007, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.003715610997460317, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-pd-cpu-model-pvc]", "lineno": 244, "outcome": "failed", "keywords": ["test_llm_inference_service[router-managed-workload-pd-cpu-model-pvc]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "pvc_storage", "__wrapped__", "pytestmark", "router-managed-workload-pd-cpu-model-pvc", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.20747978500003228, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 902.5725379260002, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1199, "message": "AssertionError: Missing true conditions: {'Ready', 'WorkloadsReady'}, expected {'RouterReady', 'Ready', 'WorkloadsReady'}, got [{'lastTransitionTime': '2026-07-07T12:39:50Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'severity': 'Info', 'status': 'False', 'type': 'PrefillWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:39:34Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-07T12:40:45Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-07T12:40:45Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'WorkloadsReady'}]"}, "traceback": [{"path": "llmisvc/test_llm_inference_service.py", "lineno": 812, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1204, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1199, "message": "AssertionError"}], "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-pd-cpu', 'model-pvc'], prompt='KServe is a', service_name='llmisvc-mod...              {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.asyncio(loop_scope=\"session\")\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-gateway-ref\",\n                        \"router-with-managed-route\",\n                        \"model-fb-opt-125m\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"custom-route-timeout-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"router-with-refs-test\",\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                            routes=[ROUTER_ROUTES[0], ROUTER_ROUTES[1]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\"router-managed\", \"workload-pd-cpu\", \"model-fb-opt-125m\"],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"custom-route-timeout-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"router-with-refs-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                    expected_gateway=ROUTER_GATEWAYS[1],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[1]],\n                            routes=[ROUTER_ROUTES[2], ROUTER_ROUTES[3]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-dp-ep-gpu\",\n                        \"workload-dp-ep-prefill-gpu\",\n                        \"model-deepseek-v2-lite\",\n                    ],\n                    prompt=\"Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically \"\n                    \"where the compute plane (P) and the data plane (D) are independently deployed and managed for a \"\n                    \"geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the \"\n                    \"fundamental challenges of network latency and data consistency, elaborate on the advanced \"\n                    \"considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: \"\n                    \"How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to \"\n                    \"evolve to support optimal performance and minimize inter-plane communication overhead, especially for \"\n                    \"synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically \"\n                    \"optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: \"\n                    \"Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) \"\n                    \"and their applicability in balancing performance and data integrity across a globally distributed data plane. \"\n                    \"Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, \"\n                    \"intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. \"\n                    \"3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently \"\n                    \"manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, \"\n                    \"cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). \"\n                    \"Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on \"\n                    \"workload patterns and data locality, potentially involving live migration strategies. \"\n                    \"4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter \"\n                    \"challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), \"\n                    \"fine-grained access control to data at rest and in motion, and identity management across disaggregated \"\n                    \"components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) \"\n                    \"concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: \"\n                    \"Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and \"\n                    \"data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) \"\n                    \"would be essential? How would incident response and troubleshooting differ in this disaggregated environment \"\n                    \"compared to traditional integrated systems? Consider the challenges of pinpointing root causes across \"\n                    \"independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries \"\n                    \"or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) \"\n                    \"where the benefits of P/D disaggregation would strongly outweigh its complexities. \"\n                    \"Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions \"\n                    \"directly interacting with object storage, in-memory disaggregation) that could further drive or \"\n                    \"transform P/D disaggregation in cloud computing.\",\n                    max_tokens=2000,\n                ),\n                marks=[\n                    pytest.mark.cluster_gpu,\n                    pytest.mark.cluster_nvidia,\n                    pytest.mark.cluster_nvidia_roce,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-no-scheduler\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"What is KServe?\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.no_scheduler,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, \"\n                    \"but without the resources requirements for DP+EP (GPUs and ROCe/IB).\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],\n            ),\n            # Scheduler config tests\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-inline-config\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-inline-config-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Chat completions endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                        \"model-qwen2.5-0.5b\",\n                    ],\n                    model_name=\"Qwen/Qwen2.5-0.5B-Instruct\",\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-configmap-ref\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-configmap-ref-test\",\n                    before_test=[create_scheduler_configmap],\n                    after_test=[delete_scheduler_configmap],\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-replicas\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-ha-replicas-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-custom-template\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-custom-template-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Scheduler v0.6 \u2192 v0.7 migration tests.\n            # Deploy v0.6-style configs and verify the controller migrates them\n            # so the v0.7 scheduler boots successfully.\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-pd-config-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-pd-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-nonzero-threshold-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-threshold-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Precise prefix KV cache routing test\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-precise-prefix-cache-inline-config\",\n                        \"workload-llmd-simulator-kvcache\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"precise-prefix-cache-test\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Models endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=create_response_assertion(with_field=\"data\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/completions\",\n                            prompt=\"KServe is a\",\n                            payload_formatter=completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/chat/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/chat/completions\",\n                            prompt=\"What is KServe?\",\n                            payload_formatter=chat_completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 LoRA adapter\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    model_name=f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\"\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/models (base + LoRA)\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=assert_models_contains(\n                        \"facebook/opt-125m\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                        \"lora-adapter-1\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # PVC storage tests -- validate direct PVC volume mount with real vLLM serving\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_multi_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_llm_inference_service(test_case: TestCase):  # noqa: F811\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        if not test_case.llm_service.metadata.annotations:\n            test_case.llm_service.metadata.annotations = {}\n    \n        test_case.llm_service.metadata.annotations[\n            \"security.opendatahub.io/enable-auth\"\n        ] = \"false\"\n        prefix = test_case.log_prefix\n    \n        test_failed = False\n        try:\n            print(f\"{prefix} Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n            print(f\"{prefix} Waiting for LLMInferenceService {service_name} to be ready\")\n>           wait_for_llm_isvc_ready(\n                kserve_client, test_case.llm_service, test_case.wait_timeout\n            )\n\nllmisvc/test_llm_inference_service.py:812: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (<kserve.api.kserve_client.KServeClient object at 0x7fcebc122b10>, {'api_version': 'serving.kserve.io/v1alpha1',\n 'kin...del-p-9d807ba3'},\n                       {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]},\n 'status': None}, 900)\nkwargs = {}, func_name = 'wait_for_llm_isvc_ready'\ntimestamp_start = '2026-07-07T12:39:21.304984', start_time = 1783427961.30528\nduration = 900.6224792003632, timestamp_end = '2026-07-07T12:54:21.927762'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fcebc122b10>\ngiven = {'api_version': 'serving.kserve.io/v1alpha1',\n 'kind': 'LLMInferenceService',\n 'metadata': {'annotations': {'security....svc-model-p-9d807ba3'},\n                       {'name': 'model-pvc-llmisvc-model-pvc-rou-49c1f027'}]},\n 'status': None}\ntimeout_seconds = 900\n\n    @log_execution\n    def wait_for_llm_isvc_ready(\n        kserve_client: KServeClient,\n        given: V1alpha1LLMInferenceService,\n        timeout_seconds: int = 900,\n    ) -> str:\n        def assert_llm_isvc_ready():\n            out = get_llmisvc(\n                kserve_client,\n                given.metadata.name,\n                given.metadata.namespace,\n                given.api_version.split(\"/\")[1],\n            )\n    \n            if \"status\" not in out:\n                raise AssertionError(\"No status found in LLM inference service\")\n    \n            status = out[\"status\"]\n            if \"conditions\" not in status:\n                raise AssertionError(\"No conditions found in status\")\n    \n            expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n            got_true_conditions = set()\n    \n            conditions = status[\"conditions\"]\n    \n            for condition in conditions:\n                if condition.get(\"status\") == \"True\":\n                    got_true_conditions.add(condition.get(\"type\"))\n    \n            missing_conditions = expected_true_conditions - got_true_conditions\n            if missing_conditions:\n                raise AssertionError(\n                    f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n                )\n            return True\n    \n>       return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0)\n\nllmisvc/test_llm_inference_service.py:1204: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7fcebc194900>\ntimeout = 900, interval = 1.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def assert_llm_isvc_ready():\n        out = get_llmisvc(\n            kserve_client,\n            given.metadata.name,\n            given.metadata.namespace,\n            given.api_version.split(\"/\")[1],\n        )\n    \n        if \"status\" not in out:\n            raise AssertionError(\"No status found in LLM inference service\")\n    \n        status = out[\"status\"]\n        if \"conditions\" not in status:\n            raise AssertionError(\"No conditions found in status\")\n    \n        expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n        got_true_conditions = set()\n    \n        conditions = status[\"conditions\"]\n    \n        for condition in conditions:\n            if condition.get(\"status\") == \"True\":\n                got_true_conditions.add(condition.get(\"type\"))\n    \n        missing_conditions = expected_true_conditions - got_true_conditions\n        if missing_conditions:\n>           raise AssertionError(\n                f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n            )\nE           AssertionError: Missing true conditions: {'Ready', 'WorkloadsReady'}, expected {'RouterReady', 'Ready', 'WorkloadsReady'}, got [{'lastTransitionTime': '2026-07-07T12:39:50Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'severity': 'Info', 'status': 'False', 'type': 'MainWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'severity': 'Info', 'status': 'False', 'type': 'PrefillWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:39:34Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-07T12:40:45Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-07T12:40:45Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:39:50Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'WorkloadsReady'}]\n\nllmisvc/test_llm_inference_service.py:1199: AssertionError"}, "teardown": {"duration": 0.004229780999594368, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-pd-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-pd-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-pd-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.653992315998039, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 254.84284076800031, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004350572002294939, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.8302639210014604, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 363.2586832809975, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0059905210000579245, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "custom_gateway", "__wrapped__", "pytestmark", "router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 2.513613234001241, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 295.4590125199975, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004696970001532463, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_multi_node-router-managed-workload-simulated-dp-ep-cpu-model-pvc]", "lineno": 244, "outcome": "failed", "keywords": ["test_llm_inference_service[router-managed-workload-simulated-dp-ep-cpu-model-pvc]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_multi_node", "pvc_storage", "__wrapped__", "pytestmark", "router-managed-workload-simulated-dp-ep-cpu-model-pvc", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.4885889750003116, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 902.5598179670014, "outcome": "failed", "crash": {"path": "/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py", "lineno": 1199, "message": "AssertionError: Missing true conditions: {'Ready', 'WorkloadsReady'}, expected {'RouterReady', 'Ready', 'WorkloadsReady'}, got [{'lastTransitionTime': '2026-07-07T12:55:13Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-07T12:55:13Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-07T12:54:43Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-07T12:55:39Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-07T12:55:39Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-07T12:55:39Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:54:43Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'severity': 'Info', 'status': 'False', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:54:43Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'status': 'False', 'type': 'WorkloadsReady'}]"}, "traceback": [{"path": "llmisvc/test_llm_inference_service.py", "lineno": 812, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1204, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1199, "message": "AssertionError"}], "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-simulated-dp-ep-cpu', 'model-pvc'], prompt='KServe is a', service_name...              {'name': 'model-pvc-llmisvc-model-pvc-rou-cfc8d654'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.asyncio(loop_scope=\"session\")\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-gateway-ref\",\n                        \"router-with-managed-route\",\n                        \"model-fb-opt-125m\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"custom-route-timeout-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs\",\n                        \"scheduler-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"router-with-refs-test\",\n                    expected_gateway=ROUTER_GATEWAYS[0],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[0]],\n                            routes=[ROUTER_ROUTES[0], ROUTER_ROUTES[1]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\"router-managed\", \"workload-pd-cpu\", \"model-fb-opt-125m\"],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-custom-route-timeout-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"custom-route-timeout-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-with-refs-pd\",\n                        \"scheduler-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"You are an expert in Kubernetes-native machine learning serving platforms, with deep knowledge of the KServe project. \"\n                    \"Explain the challenges of serving large-scale models, GPU scheduling, and how KServe integrates with capabilities like multi-model serving. \"\n                    \"Provide a detailed comparison with open source alternatives, focusing on operational trade-offs.\",\n                    service_name=\"router-with-refs-pd-test\",\n                    response_assertion=assert_200_with_choices,\n                    expected_gateway=ROUTER_GATEWAYS[1],\n                    before_test=[\n                        lambda: create_router_resources(\n                            gateways=[ROUTER_GATEWAYS[1]],\n                            routes=[ROUTER_ROUTES[2], ROUTER_ROUTES[3]],\n                        )\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.custom_gateway,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-dp-ep-gpu\",\n                        \"workload-dp-ep-prefill-gpu\",\n                        \"model-deepseek-v2-lite\",\n                    ],\n                    prompt=\"Delve into the multifaceted implications of a fully disaggregated cloud architecture, specifically \"\n                    \"where the compute plane (P) and the data plane (D) are independently deployed and managed for a \"\n                    \"geographically distributed, high-throughput, low-latency microservices ecosystem. Beyond the \"\n                    \"fundamental challenges of network latency and data consistency, elaborate on the advanced \"\n                    \"considerations and trade-offs inherent in such a setup: 1. Network Architecture and Protocols: \"\n                    \"How would the network fabric and underlying protocols (e.g., RDMA, custom transport layers) need to \"\n                    \"evolve to support optimal performance and minimize inter-plane communication overhead, especially for \"\n                    \"synchronous operations? Discuss the role of network programmability (e.g., SDN, P4) in dynamically \"\n                    \"optimizing routing and traffic flow between P and D. 2. Advanced Data Consistency and Durability: \"\n                    \"Explore sophisticated data consistency models (e.g., causal consistency, strong eventual consistency) \"\n                    \"and their applicability in balancing performance and data integrity across a globally distributed data plane. \"\n                    \"Detail strategies for ensuring data durability and fault tolerance, including multi-region replication, \"\n                    \"intelligent partitioning, and recovery mechanisms in the event of partial or full plane failures. \"\n                    \"3. Dynamic Resource Orchestration and Cost Optimization: Analyze how an orchestration layer would intelligently \"\n                    \"manage the independent scaling of compute (P) and data (D) resources, considering fluctuating workloads, \"\n                    \"cost efficiency, and performance targets (e.g., using predictive analytics for resource provisioning). \"\n                    \"Discuss mechanisms for dynamically reallocating compute nodes to different data partitions based on \"\n                    \"workload patterns and data locality, potentially involving live migration strategies. \"\n                    \"4. Security and Compliance in a Distributed Landscape: Address the enhanced security perimeter \"\n                    \"challenges, including securing communication channels between P and D (encryption in transit, mutual TLS), \"\n                    \"fine-grained access control to data at rest and in motion, and identity management across disaggregated \"\n                    \"components. Discuss how such an architecture impacts compliance with regulatory frameworks (e.g., GDPR, HIPAA) \"\n                    \"concerning data sovereignty, privacy, and auditability. 5. Operational Complexity and Observability: \"\n                    \"Examine the increased complexity in monitoring, logging, and tracing across highly decoupled compute and \"\n                    \"data planes. What specialized tooling and practices (e.g., distributed tracing with OpenTelemetry, advanced AIOps) \"\n                    \"would be essential? How would incident response and troubleshooting differ in this disaggregated environment \"\n                    \"compared to traditional integrated systems? Consider the challenges of pinpointing root causes across \"\n                    \"independent failures. 6. Real-world Applicability and Future Trends: Identify specific industries \"\n                    \"or use cases (e.g., high-frequency trading, IoT edge processing, large language model inference) \"\n                    \"where the benefits of P/D disaggregation would strongly outweigh its complexities. \"\n                    \"Conclude by speculating on emerging technologies or paradigms (e.g., serverless compute functions \"\n                    \"directly interacting with object storage, in-memory disaggregation) that could further drive or \"\n                    \"transform P/D disaggregation in cloud computing.\",\n                    max_tokens=2000,\n                ),\n                marks=[\n                    pytest.mark.cluster_gpu,\n                    pytest.mark.cluster_nvidia,\n                    pytest.mark.cluster_nvidia_roce,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-no-scheduler\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"What is KServe?\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.no_scheduler,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"This test simulates DP+EP that can run on CPU, the idea is to test the LWS-based deployment, \"\n                    \"but without the resources requirements for DP+EP (GPUs and ROCe/IB).\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],\n            ),\n            # Scheduler config tests\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-inline-config\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-inline-config-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Chat completions endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                        \"model-qwen2.5-0.5b\",\n                    ],\n                    model_name=\"Qwen/Qwen2.5-0.5B-Instruct\",\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=create_response_assertion(with_field=\"choices\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-configmap-ref\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-configmap-ref-test\",\n                    before_test=[create_scheduler_configmap],\n                    after_test=[delete_scheduler_configmap],\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-replicas\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-ha-replicas-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-custom-template\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-custom-template-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n            # Scheduler v0.6 \u2192 v0.7 migration tests.\n            # Deploy v0.6-style configs and verify the controller migrates them\n            # so the v0.7 scheduler boots successfully.\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-pd-config-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-pd-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-v06-nonzero-threshold-migration\",\n                        \"workload-llmd-simulator-pd\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"scheduler-v06-threshold-migration-test\",\n                    response_assertion=assert_200_with_choices,\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Precise prefix KV cache routing test\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"scheduler-with-precise-prefix-cache-inline-config\",\n                        \"workload-llmd-simulator-kvcache\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"precise-prefix-cache-test\",\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Models endpoint coverage\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=create_response_assertion(with_field=\"data\"),\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/completions\",\n                            prompt=\"KServe is a\",\n                            payload_formatter=completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/chat/completions\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-llmd-simulator\",\n                    ],\n                    endpoint=\"/v1/chat/completions\",\n                    prompt=\"What is KServe?\",\n                    payload_formatter=chat_completions_payload,\n                    response_assertion=assert_model_field_matches(\"facebook/opt-125m\"),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                    peers=[\n                        TestCase(\n                            base_refs=[\n                                \"router-managed\",\n                                \"workload-llmd-simulator\",\n                                \"model-qwen2.5-0.5b\",\n                            ],\n                            endpoint=\"/v1/chat/completions\",\n                            prompt=\"What is KServe?\",\n                            payload_formatter=chat_completions_payload,\n                            response_assertion=assert_model_field_matches(\n                                \"Qwen/Qwen2.5-0.5B-Instruct\"\n                            ),\n                            url_getter=get_model_routing_url,\n                            extra_headers={\n                                MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/Qwen/Qwen2.5-0.5B-Instruct\",\n                            },\n                        ),\n                    ],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.llmd_simulator,\n                    pytest.mark.model_routing,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 LoRA adapter\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/completions\",\n                    prompt=\"KServe is a\",\n                    model_name=f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    payload_formatter=completions_payload,\n                    response_assertion=assert_model_field_matches(\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\"\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # Model-based routing via X-Gateway-Model-Name header \u2014 /v1/models (base + LoRA)\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m-with-lora-hf\",\n                    ],\n                    endpoint=\"/v1/models\",\n                    response_assertion=assert_models_contains(\n                        \"facebook/opt-125m\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                        \"lora-adapter-1\",\n                        f\"publishers/{KSERVE_TEST_NAMESPACE}/models/lora-adapter-1\",\n                    ),\n                    url_getter=get_model_routing_url,\n                    extra_headers={\n                        MODEL_ROUTING_HEADER: f\"publishers/{KSERVE_TEST_NAMESPACE}/models/facebook/opt-125m\",\n                    },\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.model_routing,\n                    pytest.mark.lora,\n                ],\n            ),\n            # PVC storage tests -- validate direct PVC volume mount with real vLLM serving\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-pd-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    response_assertion=assert_200_with_choices,\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_single_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-simulated-dp-ep-cpu\",\n                        \"model-pvc\",\n                    ],\n                    prompt=\"KServe is a\",\n                    before_test=[ensure_pvc_with_model],\n                ),\n                marks=[\n                    pytest.mark.cluster_cpu,\n                    pytest.mark.cluster_multi_node,\n                    pytest.mark.pvc_storage,\n                ],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_llm_inference_service(test_case: TestCase):  # noqa: F811\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        if not test_case.llm_service.metadata.annotations:\n            test_case.llm_service.metadata.annotations = {}\n    \n        test_case.llm_service.metadata.annotations[\n            \"security.opendatahub.io/enable-auth\"\n        ] = \"false\"\n        prefix = test_case.log_prefix\n    \n        test_failed = False\n        try:\n            print(f\"{prefix} Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n            print(f\"{prefix} Waiting for LLMInferenceService {service_name} to be ready\")\n>           wait_for_llm_isvc_ready(\n                kserve_client, test_case.llm_service, test_case.wait_timeout\n            )\n\nllmisvc/test_llm_inference_service.py:812: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = (<kserve.api.kserve_client.KServeClient object at 0x7fcebc599950>, {'api_version': 'serving.kserve.io/v1alpha1',\n 'kin...pu-ll-699c687c'},\n                       {'name': 'model-pvc-llmisvc-model-pvc-rou-cfc8d654'}]},\n 'status': None}, 900)\nkwargs = {}, func_name = 'wait_for_llm_isvc_ready'\ntimestamp_start = '2026-07-07T12:54:24.627423', start_time = 1783428864.6277337\nduration = 900.5694715976715, timestamp_end = '2026-07-07T13:09:25.197207'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nkserve_client = <kserve.api.kserve_client.KServeClient object at 0x7fcebc599950>\ngiven = {'api_version': 'serving.kserve.io/v1alpha1',\n 'kind': 'LLMInferenceService',\n 'metadata': {'annotations': {'security....p-ep-cpu-ll-699c687c'},\n                       {'name': 'model-pvc-llmisvc-model-pvc-rou-cfc8d654'}]},\n 'status': None}\ntimeout_seconds = 900\n\n    @log_execution\n    def wait_for_llm_isvc_ready(\n        kserve_client: KServeClient,\n        given: V1alpha1LLMInferenceService,\n        timeout_seconds: int = 900,\n    ) -> str:\n        def assert_llm_isvc_ready():\n            out = get_llmisvc(\n                kserve_client,\n                given.metadata.name,\n                given.metadata.namespace,\n                given.api_version.split(\"/\")[1],\n            )\n    \n            if \"status\" not in out:\n                raise AssertionError(\"No status found in LLM inference service\")\n    \n            status = out[\"status\"]\n            if \"conditions\" not in status:\n                raise AssertionError(\"No conditions found in status\")\n    \n            expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n            got_true_conditions = set()\n    \n            conditions = status[\"conditions\"]\n    \n            for condition in conditions:\n                if condition.get(\"status\") == \"True\":\n                    got_true_conditions.add(condition.get(\"type\"))\n    \n            missing_conditions = expected_true_conditions - got_true_conditions\n            if missing_conditions:\n                raise AssertionError(\n                    f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n                )\n            return True\n    \n>       return wait_for(assert_llm_isvc_ready, timeout=timeout_seconds, interval=1.0)\n\nllmisvc/test_llm_inference_service.py:1204: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_llm_isvc_ready.<locals>.assert_llm_isvc_ready at 0x7fcebbc89120>\ntimeout = 900, interval = 1.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def assert_llm_isvc_ready():\n        out = get_llmisvc(\n            kserve_client,\n            given.metadata.name,\n            given.metadata.namespace,\n            given.api_version.split(\"/\")[1],\n        )\n    \n        if \"status\" not in out:\n            raise AssertionError(\"No status found in LLM inference service\")\n    \n        status = out[\"status\"]\n        if \"conditions\" not in status:\n            raise AssertionError(\"No conditions found in status\")\n    \n        expected_true_conditions = {\"Ready\", \"WorkloadsReady\", \"RouterReady\"}\n        got_true_conditions = set()\n    \n        conditions = status[\"conditions\"]\n    \n        for condition in conditions:\n            if condition.get(\"status\") == \"True\":\n                got_true_conditions.add(condition.get(\"type\"))\n    \n        missing_conditions = expected_true_conditions - got_true_conditions\n        if missing_conditions:\n>           raise AssertionError(\n                f\"Missing true conditions: {missing_conditions}, expected {expected_true_conditions}, got {conditions}\"\n            )\nE           AssertionError: Missing true conditions: {'Ready', 'WorkloadsReady'}, expected {'RouterReady', 'Ready', 'WorkloadsReady'}, got [{'lastTransitionTime': '2026-07-07T12:55:13Z', 'severity': 'Info', 'status': 'True', 'type': 'HTTPRoutesReady'}, {'lastTransitionTime': '2026-07-07T12:55:13Z', 'severity': 'Info', 'status': 'True', 'type': 'InferencePoolReady'}, {'lastTransitionTime': '2026-07-07T12:54:43Z', 'severity': 'Info', 'status': 'True', 'type': 'PresetsCombined'}, {'lastTransitionTime': '2026-07-07T12:55:39Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-07-07T12:55:39Z', 'status': 'True', 'type': 'RouterReady'}, {'lastTransitionTime': '2026-07-07T12:55:39Z', 'severity': 'Info', 'status': 'True', 'type': 'SchedulerWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:54:43Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'severity': 'Info', 'status': 'False', 'type': 'WorkerWorkloadReady'}, {'lastTransitionTime': '2026-07-07T12:54:43Z', 'message': 'LWS is progressing', 'reason': 'Progressing', 'status': 'False', 'type': 'WorkloadsReady'}]\n\nllmisvc/test_llm_inference_service.py:1199: AssertionError"}, "teardown": {"duration": 0.0028189850017952267, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-no-scheduler-workload-single-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-no-scheduler-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "no_scheduler", "__wrapped__", "pytestmark", "router-no-scheduler-workload-single-cpu-model-fb-opt-125m", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.43500081499951193, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 207.28801480699985, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004610038999089738, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_multi_node-router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_multi_node", "__wrapped__", "pytestmark", "router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.8825735370010079, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 689.846968283, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.006125553998572286, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_v1alpha1_to_v1alpha2_conversion", "lineno": 212, "outcome": "passed", "keywords": ["test_v1alpha1_to_v1alpha2_conversion", "cluster_single_node", "cluster_cpu", "pytestmark", "TestLLMInferenceServiceConversion", "conversion", "llminferenceservice", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.012653669000428636, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.42161638300240156, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.17717754000113928, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_v1alpha2_to_v1alpha1_conversion", "lineno": 303, "outcome": "passed", "keywords": ["test_v1alpha2_to_v1alpha1_conversion", "cluster_single_node", "cluster_cpu", "pytestmark", "TestLLMInferenceServiceConversion", "conversion", "llminferenceservice", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.012940677999722539, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.3919105240020144, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.020160814001428662, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_criticality_preservation_via_annotations", "lineno": 394, "outcome": "passed", "keywords": ["test_criticality_preservation_via_annotations", "cluster_single_node", "cluster_cpu", "pytestmark", "TestLLMInferenceServiceConversion", "conversion", "llminferenceservice", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.01294009699995513, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.2726508499981719, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.19587308900008793, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_lora_criticality_preservation", "lineno": 531, "outcome": "passed", "keywords": ["test_lora_criticality_preservation", "cluster_single_node", "cluster_cpu", "pytestmark", "TestLLMInferenceServiceConversion", "conversion", "llminferenceservice", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.01333851000163122, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.1864485070000228, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.2818181450020347, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_conversion.py::TestLLMInferenceServiceConversion::test_round_trip_conversion_preserves_fields", "lineno": 680, "outcome": "passed", "keywords": ["test_round_trip_conversion_preserves_fields", "cluster_single_node", "cluster_cpu", "pytestmark", "TestLLMInferenceServiceConversion", "conversion", "llminferenceservice", "test_llm_inference_service_conversion.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.013465743999404367, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 0.29736209999828134, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.10737191699809046, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service_stop.py::test_llm_stop_feature[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 39, "outcome": "passed", "keywords": ["test_llm_stop_feature[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "test_llm_inference_service_stop.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.38442887000201154, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 430.8820468039994, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004419791999680456, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-inline-config-workload-llmd-simulator]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-inline-config-workload-llmd-simulator]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-inline-config-workload-llmd-simulator", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.6208693730004597, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 154.14990598600343, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.0034016209974652156, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-workload-llmd-simulator-model-qwen2.5-0.5b]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-workload-llmd-simulator-model-qwen2.5-0.5b]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-workload-llmd-simulator-model-qwen2.5-0.5b", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.5389404810011911, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 154.18601922000016, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004443762998562306, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-configmap-ref-workload-llmd-simulator]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-configmap-ref-workload-llmd-simulator]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-configmap-ref-workload-llmd-simulator", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.9106840429994918, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 172.67747119200067, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.059828099998412654, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_lora_adapters.py::test_llm_with_lora_adapters[cluster_cpu-single-lora-adapter-hf]", "lineno": 203, "outcome": "passed", "keywords": ["test_llm_with_lora_adapters[single-lora-adapter-hf]", "parametrize", "llminferenceservice", "cluster_cpu", "lora", "__wrapped__", "pytestmark", "single-lora-adapter-hf", "test_llm_lora_adapters.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.00041453299854765646, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 260.4483340020015, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004609217998222448, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-replicas-workload-llmd-simulator]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-replicas-workload-llmd-simulator]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-replicas-workload-llmd-simulator", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.42748709800071083, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 154.566355166, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004476213001908036, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_lora_adapters.py::test_llm_with_lora_adapters[cluster_cpu-multiple-lora-adapters]", "lineno": 203, "outcome": "passed", "keywords": ["test_llm_with_lora_adapters[multiple-lora-adapters]", "parametrize", "llminferenceservice", "cluster_cpu", "lora", "__wrapped__", "pytestmark", "multiple-lora-adapters", "test_llm_lora_adapters.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.00043440299850772135, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 239.58771535300184, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.007323990001168568, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-with-custom-template-workload-llmd-simulator]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-with-custom-template-workload-llmd-simulator]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-scheduler-with-custom-template-workload-llmd-simulator", "llmd_simulator", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.5162048240017612, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 199.68350317699878, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.00393445800000336, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_tls.py::test_llm_tls_resources[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 92, "outcome": "passed", "keywords": ["test_llm_tls_resources[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "test_llm_tls.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.20325123500151676, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 245.85965348100217, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.003956888998800423, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.39334357699772227, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 177.04760114999954, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.004642829000658821, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_llm_inference_service.py::test_llm_inference_service[cluster_cpu-cluster_single_node-router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd]", "lineno": 244, "outcome": "passed", "keywords": ["test_llm_inference_service[router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd]", "parametrize", "asyncio", "llminferenceservice", "cluster_cpu", "cluster_single_node", "llmd_simulator", "__wrapped__", "pytestmark", "router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd", "test_llm_inference_service.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 0.4375621670005785, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 239.24460473200088, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "teardown": {"duration": 0.009852264996879967, "outcome": "passed", "longrepr": "[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}, {"nodeid": "llmisvc/test_prestop_hook.py::test_prestop_hook[cluster_cpu-cluster_single_node-router-managed-workload-single-cpu-model-fb-opt-125m]", "lineno": 40, "outcome": "failed", "keywords": ["test_prestop_hook[router-managed-workload-single-cpu-model-fb-opt-125m]", "parametrize", "llminferenceservice", "cluster_cpu", "cluster_single_node", "__wrapped__", "pytestmark", "router-managed-workload-single-cpu-model-fb-opt-125m", "test_prestop_hook.py", "llmisvc/__init__.py", "e2e"], "setup": {"duration": 1.8066686099991784, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}, "call": {"duration": 940.4026964539989, "outcome": "failed", "crash": {"path": "/workspace/source/python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py", "lineno": 519, "message": "urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443): Max retries exceeded with url: /api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload (Caused by NameResolutionError(\"HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443): Failed to resolve 'a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com' ([Errno -2] Name or service not known)\"))"}, "traceback": [{"path": "llmisvc/test_prestop_hook.py", "lineno": 79, "message": ""}, {"path": "llmisvc/logging.py", "lineno": 40, "message": ""}, {"path": "llmisvc/test_prestop_hook.py", "lineno": 149, "message": ""}, {"path": "llmisvc/test_llm_inference_service.py", "lineno": 1215, "message": ""}, {"path": "llmisvc/test_prestop_hook.py", "lineno": 139, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/core_v1_api.py", "lineno": 15968, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/core_v1_api.py", "lineno": 16087, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py", "lineno": 348, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py", "lineno": 180, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py", "lineno": 373, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py", "lineno": 244, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py", "lineno": 217, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/_request_methods.py", "lineno": 135, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/_request_methods.py", "lineno": 182, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/poolmanager.py", "lineno": 457, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py", "lineno": 871, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py", "lineno": 871, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py", "lineno": 871, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py", "lineno": 841, "message": ""}, {"path": "../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py", "lineno": 519, "message": "MaxRetryError"}], "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python\n\nself = <HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443) at 0x7fcebc079110>\n\n    def _new_conn(self) -> socket.socket:\n        \"\"\"Establish a socket connection and set nodelay settings on it.\n    \n        :return: New socket connection.\n        \"\"\"\n        try:\n>           sock = connection.create_connection(\n                (self._dns_host, self.port),\n                self.timeout,\n                source_address=self.source_address,\n                socket_options=self.socket_options,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:204: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\naddress = ('a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', 6443)\ntimeout = None, source_address = None, socket_options = [(6, 1, 1)]\n\n    def create_connection(\n        address: tuple[str, int],\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        source_address: tuple[str, int] | None = None,\n        socket_options: _TYPE_SOCKET_OPTIONS | None = None,\n    ) -> socket.socket:\n        \"\"\"Connect to *address* and return the socket object.\n    \n        Convenience function.  Connect to *address* (a 2-tuple ``(host,\n        port)``) and return the socket object.  Passing the optional\n        *timeout* parameter will set the timeout on the socket instance\n        before attempting to connect.  If no *timeout* is supplied, the\n        global default timeout setting returned by :func:`socket.getdefaulttimeout`\n        is used.  If *source_address* is set it must be a tuple of (host, port)\n        for the socket to bind as a source address before making the connection.\n        An host of '' or port 0 tells the OS to use the default.\n        \"\"\"\n    \n        host, port = address\n        if host.startswith(\"[\"):\n            host = host.strip(\"[]\")\n        err = None\n    \n        # Using the value from allowed_gai_family() in the context of getaddrinfo lets\n        # us select whether to work with IPv4 DNS records, IPv6 records, or both.\n        # The original create_connection function always returns all records.\n        family = allowed_gai_family()\n    \n        try:\n            host.encode(\"idna\")\n        except UnicodeError:\n            raise LocationParseError(f\"'{host}', label empty or too long\") from None\n    \n>       for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/connection.py:60: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nhost = 'a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com'\nport = 6443, family = <AddressFamily.AF_UNSPEC: 0>\ntype = <SocketKind.SOCK_STREAM: 1>, proto = 0, flags = 0\n\n    def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):\n        \"\"\"Resolve host and port into list of address info entries.\n    \n        Translate the host/port argument into a sequence of 5-tuples that contain\n        all the necessary arguments for creating a socket connected to that service.\n        host is a domain name, a string representation of an IPv4/v6 address or\n        None. port is a string service name such as 'http', a numeric port number or\n        None. By passing None as the value of host and port, you can pass NULL to\n        the underlying C API.\n    \n        The family, type and proto arguments can be optionally specified in order to\n        narrow the list of addresses returned. Passing zero as a value for each of\n        these arguments selects the full range of results.\n        \"\"\"\n        # We override this function since we want to translate the numeric family\n        # and socket type values to enum constants.\n        addrlist = []\n>       for res in _socket.getaddrinfo(host, port, family, type, proto, flags):\nE       socket.gaierror: [Errno -2] Name or service not known\n\n/usr/lib64/python3.11/socket.py:974: gaierror\n\nThe above exception was the direct cause of the following exception:\n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False, timeout = None, pool_timeout = None\nrelease_conn = True, chunked = False, body_pos = None, preload_content = True\ndecode_content = True, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/api/v1/namespaces/kserve-ci-e2e-test/pods', query='labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload', fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n>           response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:787: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nconn = <HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443) at 0x7fcebc079110>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=None, read=None, total=None), chunked = False\nresponse_conn = None, preload_content = True, decode_content = True\nenforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n                self._validate_conn(conn)\n            except (SocketTimeout, BaseSSLError) as e:\n                self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)\n                raise\n    \n        # _validate_conn() starts the connection to an HTTPS proxy\n        # so we need to wrap errors with 'ProxyError' here too.\n        except (\n            OSError,\n            NewConnectionError,\n            TimeoutError,\n            BaseSSLError,\n            CertificateError,\n            SSLError,\n        ) as e:\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            # If the connection didn't successfully connect to it's proxy\n            # then there\n            if isinstance(\n                new_e, (OSError, NewConnectionError, TimeoutError, SSLError)\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n>           raise new_e\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:488: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nconn = <HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443) at 0x7fcebc079110>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\ntimeout = Timeout(connect=None, read=None, total=None), chunked = False\nresponse_conn = None, preload_content = True, decode_content = True\nenforce_content_length = True\n\n    def _make_request(\n        self,\n        conn: BaseHTTPConnection,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | None = None,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        chunked: bool = False,\n        response_conn: BaseHTTPConnection | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        enforce_content_length: bool = True,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Perform a request on a given urllib connection object taken from our\n        pool.\n    \n        :param conn:\n            a connection from one of our connection pools\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            Pass ``None`` to retry until you receive a response. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param response_conn:\n            Set this to ``None`` if you will handle releasing the connection or\n            set the connection to have the response release it.\n    \n        :param preload_content:\n          If True, the response's body will be preloaded during construction.\n    \n        :param decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param enforce_content_length:\n            Enforce content length checking. Body returned by server must match\n            value of Content-Length header, if present. Otherwise, raise error.\n        \"\"\"\n        self.num_requests += 1\n    \n        timeout_obj = self._get_timeout(timeout)\n        timeout_obj.start_connect()\n        conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)\n    \n        try:\n            # Trigger any extra validation we need to do.\n            try:\n>               self._validate_conn(conn)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:464: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nconn = <HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443) at 0x7fcebc079110>\n\n    def _validate_conn(self, conn: BaseHTTPConnection) -> None:\n        \"\"\"\n        Called right before a request is made, after the socket is created.\n        \"\"\"\n        super()._validate_conn(conn)\n    \n        # Force connect early to allow us to validate the connection.\n        if conn.is_closed:\n>           conn.connect()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1093: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443) at 0x7fcebc079110>\n\n    def connect(self) -> None:\n        # Today we don't need to be doing this step before the /actual/ socket\n        # connection, however in the future we'll need to decide whether to\n        # create a new socket or re-use an existing \"shared\" socket as a part\n        # of the HTTP/2 handshake dance.\n        if self._tunnel_host is not None and self._tunnel_port is not None:\n            probe_http2_host = self._tunnel_host\n            probe_http2_port = self._tunnel_port\n        else:\n            probe_http2_host = self.host\n            probe_http2_port = self.port\n    \n        # Check if the target origin supports HTTP/2.\n        # If the value comes back as 'None' it means that the current thread\n        # is probing for HTTP/2 support. Otherwise, we're waiting for another\n        # probe to complete, or we get a value right away.\n        target_supports_http2: bool | None\n        if \"h2\" in ssl_.ALPN_PROTOCOLS:\n            target_supports_http2 = http2_probe.acquire_and_get(\n                host=probe_http2_host, port=probe_http2_port\n            )\n        else:\n            # If HTTP/2 isn't going to be offered it doesn't matter if\n            # the target supports HTTP/2. Don't want to make a probe.\n            target_supports_http2 = False\n    \n        if self._connect_callback is not None:\n            self._connect_callback(\n                \"before connect\",\n                thread_id=threading.get_ident(),\n                target_supports_http2=target_supports_http2,\n            )\n    \n        try:\n            sock: socket.socket | ssl.SSLSocket\n>           self.sock = sock = self._new_conn()\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:759: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443) at 0x7fcebc079110>\n\n    def _new_conn(self) -> socket.socket:\n        \"\"\"Establish a socket connection and set nodelay settings on it.\n    \n        :return: New socket connection.\n        \"\"\"\n        try:\n            sock = connection.create_connection(\n                (self._dns_host, self.port),\n                self.timeout,\n                source_address=self.source_address,\n                socket_options=self.socket_options,\n            )\n        except socket.gaierror as e:\n>           raise NameResolutionError(self.host, self, e) from e\nE           urllib3.exceptions.NameResolutionError: HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443): Failed to resolve 'a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com' ([Errno -2] Name or service not known)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connection.py:211: NameResolutionError\n\nThe above exception was the direct cause of the following exception:\n\ntest_case = TestCase(base_refs=['router-managed', 'workload-single-cpu', 'model-fb-opt-125m'], prompt='KServe is a', service_name=...               {'name': 'model-fb-opt-125m-prestop-hook-584a80fb'}]},\n 'status': None}, model_name='facebook/opt-125m')\n\n    @pytest.mark.llminferenceservice\n    @pytest.mark.parametrize(\n        \"test_case\",\n        [\n            pytest.param(\n                TestCase(\n                    base_refs=[\n                        \"router-managed\",\n                        \"workload-single-cpu\",\n                        \"model-fb-opt-125m\",\n                    ],\n                    prompt=\"KServe is a\",\n                    service_name=\"prestop-hook-test\",\n                ),\n                marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node],\n            ),\n        ],\n        indirect=[\"test_case\"],\n        ids=generate_test_id,\n    )\n    @log_execution\n    def test_prestop_hook(test_case: TestCase):\n        \"\"\"Verify the preStop lifecycle hook delays pod termination by at least 15 seconds.\"\"\"\n        inject_k8s_proxy()\n    \n        kserve_client = KServeClient(\n            config_file=os.environ.get(\"KUBECONFIG\", \"~/.kube/config\"),\n            client_configuration=client.Configuration(),\n        )\n    \n        service_name = test_case.llm_service.metadata.name\n        namespace = test_case.llm_service.metadata.namespace\n        test_failed = False\n    \n        try:\n            print(f\"Creating LLMInferenceService {service_name}\")\n            create_llmisvc(kserve_client, test_case.llm_service)\n    \n>           workload_pod_name = wait_for_workload_pod_running(\n                namespace, service_name, timeout_seconds=test_case.wait_timeout\n            )\n\nllmisvc/test_prestop_hook.py:79: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nargs = ('kserve-ci-e2e-test', 'prestop-hook-test')\nkwargs = {'timeout_seconds': 900}, func_name = 'wait_for_workload_pod_running'\ntimestamp_start = '2026-07-07T13:29:09.327942', start_time = 1783430949.3280036\nduration = 939.7024481296539, timestamp_end = '2026-07-07T13:44:49.030455'\n\n    @functools.wraps(func)\n    def wrapper(*args, **kwargs):\n        func_name = func.__name__\n    \n        timestamp_start = datetime.now().isoformat()\n        logger.info(\n            f\"[{func_name}] [{timestamp_start}] start - args={args}, kwargs={kwargs}\"\n        )\n        start_time = time.time()\n    \n        try:\n>           result = func(*args, **kwargs)\n\nllmisvc/logging.py:40: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nnamespace = 'kserve-ci-e2e-test', service_name = 'prestop-hook-test'\ntimeout_seconds = 900\n\n    @log_execution\n    def wait_for_workload_pod_running(\n        namespace: str,\n        service_name: str,\n        timeout_seconds: int = 900,\n    ) -> str:\n        \"\"\"Wait for the main workload pod to reach Running state, return its name.\"\"\"\n        core_v1 = client.CoreV1Api()\n        label_selector = (\n            f\"app.kubernetes.io/name={service_name},kserve.io/component=workload\"\n        )\n    \n        def assert_pod_running() -> str:\n            pods = core_v1.list_namespaced_pod(namespace, label_selector=label_selector)\n            running = [\n                p\n                for p in pods.items\n                if p.status.phase == \"Running\"\n                and all(cs.ready for cs in (p.status.container_statuses or []))\n            ]\n            assert running, f\"No Running pods found for {label_selector} in {namespace}\"\n            return running[0].metadata.name\n    \n>       return wait_for(assert_pod_running, timeout=timeout_seconds, interval=5.0)\n\nllmisvc/test_prestop_hook.py:149: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nassertion_fn = <function wait_for_workload_pod_running.<locals>.assert_pod_running at 0x7fcebbbf1c60>\ntimeout = 900, interval = 5.0\n\n    def wait_for(\n        assertion_fn: Callable[[], Any], timeout: float = 5.0, interval: float = 0.1\n    ) -> Any:\n        \"\"\"Wait for the assertion to succeed within timeout.\"\"\"\n        deadline = time.time() + timeout\n        last_msg = None\n        while True:\n            try:\n>               return assertion_fn()\n\nllmisvc/test_llm_inference_service.py:1215: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\n    def assert_pod_running() -> str:\n>       pods = core_v1.list_namespaced_pod(namespace, label_selector=label_selector)\n\nllmisvc/test_prestop_hook.py:139: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.api.core_v1_api.CoreV1Api object at 0x7fcebc031f90>\nnamespace = 'kserve-ci-e2e-test'\nkwargs = {'_return_http_data_only': True, 'label_selector': 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload'}\n\n    def list_namespaced_pod(self, namespace, **kwargs):  # noqa: E501\n        \"\"\"list_namespaced_pod  # noqa: E501\n    \n        list or watch objects of kind Pod  # noqa: E501\n        This method makes a synchronous HTTP request by default. To make an\n        asynchronous HTTP request, please pass async_req=True\n        >>> thread = api.list_namespaced_pod(namespace, async_req=True)\n        >>> result = thread.get()\n    \n        :param async_req bool: execute request asynchronously\n        :param str namespace: object name and auth scope, such as for teams and projects (required)\n        :param str pretty: If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).\n        :param bool allow_watch_bookmarks: allowWatchBookmarks requests watch events with type \\\"BOOKMARK\\\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored.\n        :param str _continue: The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \\\"next key\\\".  This field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.\n        :param str field_selector: A selector to restrict the list of returned objects by their fields. Defaults to everything.\n        :param str label_selector: A selector to restrict the list of returned objects by their labels. Defaults to everything.\n        :param int limit: limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.  The server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.\n        :param str resource_version: resourceVersion sets a constraint on what resource versions a request may be served from. See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.  Defaults to unset\n        :param str resource_version_match: resourceVersionMatch determines how resourceVersion is applied to list calls. It is highly recommended that resourceVersionMatch be set for list calls where resourceVersion is set See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.  Defaults to unset\n        :param bool send_initial_events: `sendInitialEvents=true` may be set together with `watch=true`. In that case, the watch stream will begin with synthetic events to produce the current state of objects in the collection. Once all such events have been sent, a synthetic \\\"Bookmark\\\" event  will be sent. The bookmark will report the ResourceVersion (RV) corresponding to the set of objects, and be marked with `\\\"k8s.io/initial-events-end\\\": \\\"true\\\"` annotation. Afterwards, the watch stream will proceed as usual, sending watch events corresponding to changes (subsequent to the RV) to objects watched.  When `sendInitialEvents` option is set, we require `resourceVersionMatch` option to also be set. The semantic of the watch request is as following: - `resourceVersionMatch` = NotOlderThan   is interpreted as \\\"data at least as new as the provided `resourceVersion`\\\"   and the bookmark event is send when the state is synced   to a `resourceVersion` at least as fresh as the one provided by the ListOptions.   If `resourceVersion` is unset, this is interpreted as \\\"consistent read\\\" and the   bookmark event is send when the state is synced at least to the moment   when request started being processed. - `resourceVersionMatch` set to any other value or unset   Invalid error is returned.  Defaults to true if `resourceVersion=\\\"\\\"` or `resourceVersion=\\\"0\\\"` (for backward compatibility reasons) and to false otherwise.\n        :param int timeout_seconds: Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.\n        :param bool watch: Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.\n        :param _preload_content: if False, the urllib3.HTTPResponse object will\n                                 be returned without reading/decoding response\n                                 data. Default is True.\n        :param _request_timeout: timeout setting for this request. If one\n                                 number provided, it will be total request\n                                 timeout. It can also be a pair (tuple) of\n                                 (connection, read) timeouts.\n        :return: V1PodList\n                 If the method is called asynchronously,\n                 returns the request thread.\n        \"\"\"\n        kwargs['_return_http_data_only'] = True\n>       return self.list_namespaced_pod_with_http_info(namespace, **kwargs)  # noqa: E501\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/core_v1_api.py:15968: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.api.core_v1_api.CoreV1Api object at 0x7fcebc031f90>\nnamespace = 'kserve-ci-e2e-test'\nkwargs = {'_return_http_data_only': True, 'label_selector': 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload'}\nlocal_var_params = {'_return_http_data_only': True, 'all_params': ['namespace', 'pretty', 'allow_watch_bookmarks', '_continue', 'field_selector', 'label_selector', ...], 'auth_settings': ['BearerToken'], 'body_params': None, ...}\nall_params = ['namespace', 'pretty', 'allow_watch_bookmarks', '_continue', 'field_selector', 'label_selector', ...]\nkey = '_return_http_data_only', val = True, collection_formats = {}\npath_params = {'namespace': 'kserve-ci-e2e-test'}\nquery_params = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\n\n    def list_namespaced_pod_with_http_info(self, namespace, **kwargs):  # noqa: E501\n        \"\"\"list_namespaced_pod  # noqa: E501\n    \n        list or watch objects of kind Pod  # noqa: E501\n        This method makes a synchronous HTTP request by default. To make an\n        asynchronous HTTP request, please pass async_req=True\n        >>> thread = api.list_namespaced_pod_with_http_info(namespace, async_req=True)\n        >>> result = thread.get()\n    \n        :param async_req bool: execute request asynchronously\n        :param str namespace: object name and auth scope, such as for teams and projects (required)\n        :param str pretty: If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).\n        :param bool allow_watch_bookmarks: allowWatchBookmarks requests watch events with type \\\"BOOKMARK\\\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored.\n        :param str _continue: The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \\\"next key\\\".  This field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.\n        :param str field_selector: A selector to restrict the list of returned objects by their fields. Defaults to everything.\n        :param str label_selector: A selector to restrict the list of returned objects by their labels. Defaults to everything.\n        :param int limit: limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.  The server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.\n        :param str resource_version: resourceVersion sets a constraint on what resource versions a request may be served from. See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.  Defaults to unset\n        :param str resource_version_match: resourceVersionMatch determines how resourceVersion is applied to list calls. It is highly recommended that resourceVersionMatch be set for list calls where resourceVersion is set See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.  Defaults to unset\n        :param bool send_initial_events: `sendInitialEvents=true` may be set together with `watch=true`. In that case, the watch stream will begin with synthetic events to produce the current state of objects in the collection. Once all such events have been sent, a synthetic \\\"Bookmark\\\" event  will be sent. The bookmark will report the ResourceVersion (RV) corresponding to the set of objects, and be marked with `\\\"k8s.io/initial-events-end\\\": \\\"true\\\"` annotation. Afterwards, the watch stream will proceed as usual, sending watch events corresponding to changes (subsequent to the RV) to objects watched.  When `sendInitialEvents` option is set, we require `resourceVersionMatch` option to also be set. The semantic of the watch request is as following: - `resourceVersionMatch` = NotOlderThan   is interpreted as \\\"data at least as new as the provided `resourceVersion`\\\"   and the bookmark event is send when the state is synced   to a `resourceVersion` at least as fresh as the one provided by the ListOptions.   If `resourceVersion` is unset, this is interpreted as \\\"consistent read\\\" and the   bookmark event is send when the state is synced at least to the moment   when request started being processed. - `resourceVersionMatch` set to any other value or unset   Invalid error is returned.  Defaults to true if `resourceVersion=\\\"\\\"` or `resourceVersion=\\\"0\\\"` (for backward compatibility reasons) and to false otherwise.\n        :param int timeout_seconds: Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.\n        :param bool watch: Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.\n        :param _return_http_data_only: response data without head status code\n                                       and headers\n        :param _preload_content: if False, the urllib3.HTTPResponse object will\n                                 be returned without reading/decoding response\n                                 data. Default is True.\n        :param _request_timeout: timeout setting for this request. If one\n                                 number provided, it will be total request\n                                 timeout. It can also be a pair (tuple) of\n                                 (connection, read) timeouts.\n        :return: tuple(V1PodList, status_code(int), headers(HTTPHeaderDict))\n                 If the method is called asynchronously,\n                 returns the request thread.\n        \"\"\"\n    \n        local_var_params = locals()\n    \n        all_params = [\n            'namespace',\n            'pretty',\n            'allow_watch_bookmarks',\n            '_continue',\n            'field_selector',\n            'label_selector',\n            'limit',\n            'resource_version',\n            'resource_version_match',\n            'send_initial_events',\n            'timeout_seconds',\n            'watch'\n        ]\n        all_params.extend(\n            [\n                'async_req',\n                '_return_http_data_only',\n                '_preload_content',\n                '_request_timeout'\n            ]\n        )\n    \n        for key, val in six.iteritems(local_var_params['kwargs']):\n            if key not in all_params:\n                raise ApiTypeError(\n                    \"Got an unexpected keyword argument '%s'\"\n                    \" to method list_namespaced_pod\" % key\n                )\n            local_var_params[key] = val\n        del local_var_params['kwargs']\n        # verify the required parameter 'namespace' is set\n        if self.api_client.client_side_validation and ('namespace' not in local_var_params or  # noqa: E501\n                                                        local_var_params['namespace'] is None):  # noqa: E501\n            raise ApiValueError(\"Missing the required parameter `namespace` when calling `list_namespaced_pod`\")  # noqa: E501\n    \n        collection_formats = {}\n    \n        path_params = {}\n        if 'namespace' in local_var_params:\n            path_params['namespace'] = local_var_params['namespace']  # noqa: E501\n    \n        query_params = []\n        if 'pretty' in local_var_params and local_var_params['pretty'] is not None:  # noqa: E501\n            query_params.append(('pretty', local_var_params['pretty']))  # noqa: E501\n        if 'allow_watch_bookmarks' in local_var_params and local_var_params['allow_watch_bookmarks'] is not None:  # noqa: E501\n            query_params.append(('allowWatchBookmarks', local_var_params['allow_watch_bookmarks']))  # noqa: E501\n        if '_continue' in local_var_params and local_var_params['_continue'] is not None:  # noqa: E501\n            query_params.append(('continue', local_var_params['_continue']))  # noqa: E501\n        if 'field_selector' in local_var_params and local_var_params['field_selector'] is not None:  # noqa: E501\n            query_params.append(('fieldSelector', local_var_params['field_selector']))  # noqa: E501\n        if 'label_selector' in local_var_params and local_var_params['label_selector'] is not None:  # noqa: E501\n            query_params.append(('labelSelector', local_var_params['label_selector']))  # noqa: E501\n        if 'limit' in local_var_params and local_var_params['limit'] is not None:  # noqa: E501\n            query_params.append(('limit', local_var_params['limit']))  # noqa: E501\n        if 'resource_version' in local_var_params and local_var_params['resource_version'] is not None:  # noqa: E501\n            query_params.append(('resourceVersion', local_var_params['resource_version']))  # noqa: E501\n        if 'resource_version_match' in local_var_params and local_var_params['resource_version_match'] is not None:  # noqa: E501\n            query_params.append(('resourceVersionMatch', local_var_params['resource_version_match']))  # noqa: E501\n        if 'send_initial_events' in local_var_params and local_var_params['send_initial_events'] is not None:  # noqa: E501\n            query_params.append(('sendInitialEvents', local_var_params['send_initial_events']))  # noqa: E501\n        if 'timeout_seconds' in local_var_params and local_var_params['timeout_seconds'] is not None:  # noqa: E501\n            query_params.append(('timeoutSeconds', local_var_params['timeout_seconds']))  # noqa: E501\n        if 'watch' in local_var_params and local_var_params['watch'] is not None:  # noqa: E501\n            query_params.append(('watch', local_var_params['watch']))  # noqa: E501\n    \n        header_params = {}\n    \n        form_params = []\n        local_var_files = {}\n    \n        body_params = None\n        # HTTP header `Accept`\n        header_params['Accept'] = self.api_client.select_header_accept(\n            ['application/json', 'application/yaml', 'application/vnd.kubernetes.protobuf', 'application/cbor', 'application/json;stream=watch', 'application/vnd.kubernetes.protobuf;stream=watch', 'application/cbor-seq'])  # noqa: E501\n    \n        # Authentication setting\n        auth_settings = ['BearerToken']  # noqa: E501\n    \n>       return self.api_client.call_api(\n            '/api/v1/namespaces/{namespace}/pods', 'GET',\n            path_params,\n            query_params,\n            header_params,\n            body=body_params,\n            post_params=form_params,\n            files=local_var_files,\n            response_type='V1PodList',  # noqa: E501\n            auth_settings=auth_settings,\n            async_req=local_var_params.get('async_req'),\n            _return_http_data_only=local_var_params.get('_return_http_data_only'),  # noqa: E501\n            _preload_content=local_var_params.get('_preload_content', True),\n            _request_timeout=local_var_params.get('_request_timeout'),\n            collection_formats=collection_formats)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/core_v1_api.py:16087: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.api_client.ApiClient object at 0x7fcebc033bd0>\nresource_path = '/api/v1/namespaces/{namespace}/pods', method = 'GET'\npath_params = {'namespace': 'kserve-ci-e2e-test'}\nquery_params = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\nheader_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nbody = None, post_params = [], files = {}, response_type = 'V1PodList'\nauth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True\ncollection_formats = {}, _preload_content = True, _request_timeout = None\n_host = None\n\n    def call_api(self, resource_path, method,\n                 path_params=None, query_params=None, header_params=None,\n                 body=None, post_params=None, files=None,\n                 response_type=None, auth_settings=None, async_req=None,\n                 _return_http_data_only=None, collection_formats=None,\n                 _preload_content=True, _request_timeout=None, _host=None):\n        \"\"\"Makes the HTTP request (synchronous) and returns deserialized data.\n    \n        To make an async_req request, set the async_req parameter.\n    \n        :param resource_path: Path to method endpoint.\n        :param method: Method to call.\n        :param path_params: Path parameters in the url.\n        :param query_params: Query parameters in the url.\n        :param header_params: Header parameters to be\n            placed in the request header.\n        :param body: Request body.\n        :param post_params dict: Request post form parameters,\n            for `application/x-www-form-urlencoded`, `multipart/form-data`.\n        :param auth_settings list: Auth Settings names for the request.\n        :param response: Response data type.\n        :param files dict: key -> filename, value -> filepath,\n            for `multipart/form-data`.\n        :param async_req bool: execute request asynchronously\n        :param _return_http_data_only: response data without head status code\n                                       and headers\n        :param collection_formats: dict of collection formats for path, query,\n            header, and post parameters.\n        :param _preload_content: if False, the urllib3.HTTPResponse object will\n                                 be returned without reading/decoding response\n                                 data. Default is True.\n        :param _request_timeout: timeout setting for this request. If one\n                                 number provided, it will be total request\n                                 timeout. It can also be a pair (tuple) of\n                                 (connection, read) timeouts.\n        :return:\n            If async_req parameter is True,\n            the request will be called asynchronously.\n            The method will return the request thread.\n            If parameter async_req is False or missing,\n            then the method will return the response directly.\n        \"\"\"\n        if not async_req:\n>           return self.__call_api(resource_path, method,\n                                   path_params, query_params, header_params,\n                                   body, post_params, files,\n                                   response_type, auth_settings,\n                                   _return_http_data_only, collection_formats,\n                                   _preload_content, _request_timeout, _host)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.api_client.ApiClient object at 0x7fcebc033bd0>\nresource_path = '/api/v1/namespaces/kserve-ci-e2e-test/pods', method = 'GET'\npath_params = [('namespace', 'kserve-ci-e2e-test')]\nquery_params = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\nheader_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nbody = None, post_params = [], files = {}, response_type = 'V1PodList'\nauth_settings = ['BearerToken'], _return_http_data_only = True\ncollection_formats = {}, _preload_content = True, _request_timeout = None\n_host = None\n\n    def __call_api(\n            self, resource_path, method, path_params=None,\n            query_params=None, header_params=None, body=None, post_params=None,\n            files=None, response_type=None, auth_settings=None,\n            _return_http_data_only=None, collection_formats=None,\n            _preload_content=True, _request_timeout=None, _host=None):\n    \n        config = self.configuration\n    \n        # header parameters\n        header_params = header_params or {}\n        header_params.update(self.default_headers)\n        if self.cookie:\n            header_params['Cookie'] = self.cookie\n        if header_params:\n            header_params = self.sanitize_for_serialization(header_params)\n            header_params = dict(self.parameters_to_tuples(header_params,\n                                                           collection_formats))\n    \n        # path parameters\n        if path_params:\n            path_params = self.sanitize_for_serialization(path_params)\n            path_params = self.parameters_to_tuples(path_params,\n                                                    collection_formats)\n            for k, v in path_params:\n                # specified safe chars, encode everything\n                resource_path = resource_path.replace(\n                    '{%s}' % k,\n                    quote(str(v), safe=config.safe_chars_for_path_param)\n                )\n    \n        # query parameters\n        if query_params:\n            query_params = self.sanitize_for_serialization(query_params)\n            query_params = self.parameters_to_tuples(query_params,\n                                                     collection_formats)\n    \n        # post parameters\n        if post_params or files:\n            post_params = post_params if post_params else []\n            post_params = self.sanitize_for_serialization(post_params)\n            post_params = self.parameters_to_tuples(post_params,\n                                                    collection_formats)\n            post_params.extend(self.files_parameters(files))\n    \n        # auth setting\n        self.update_params_for_auth(header_params, query_params, auth_settings)\n    \n        # body\n        if body:\n            body = self.sanitize_for_serialization(body)\n    \n        # request url\n        if _host is None:\n            url = self.configuration.host + resource_path\n        else:\n            # use server/host defined in path or operation instead\n            url = _host + resource_path\n    \n        # perform request and return response\n>       response_data = self.request(\n            method, url, query_params=query_params, headers=header_params,\n            post_params=post_params, body=body,\n            _preload_content=_preload_content,\n            _request_timeout=_request_timeout)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.api_client.ApiClient object at 0x7fcebc033bd0>\nmethod = 'GET'\nurl = 'https://a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/kserve-ci-e2e-test/pods'\nquery_params = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\npost_params = [], body = None, _preload_content = True, _request_timeout = None\n\n    def request(self, method, url, query_params=None, headers=None,\n                post_params=None, body=None, _preload_content=True,\n                _request_timeout=None):\n        \"\"\"Makes the HTTP request using RESTClient.\"\"\"\n        if method == \"GET\":\n>           return self.rest_client.GET(url,\n                                        query_params=query_params,\n                                        _preload_content=_preload_content,\n                                        _request_timeout=_request_timeout,\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:373: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.rest.RESTClientObject object at 0x7fcebc033b10>\nurl = 'https://a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/kserve-ci-e2e-test/pods'\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nquery_params = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\n_preload_content = True, _request_timeout = None\n\n    def GET(self, url, headers=None, query_params=None, _preload_content=True,\n            _request_timeout=None):\n>       return self.request(\"GET\", url,\n                            headers=headers,\n                            _preload_content=_preload_content,\n                            _request_timeout=_request_timeout,\n                            query_params=query_params)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:244: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <kubernetes.client.rest.RESTClientObject object at 0x7fcebc033b10>\nmethod = 'GET'\nurl = 'https://a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/kserve-ci-e2e-test/pods'\nquery_params = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nbody = None, post_params = {}, _preload_content = True, _request_timeout = None\n\n    def request(self, method, url, query_params=None, headers=None,\n                body=None, post_params=None, _preload_content=True,\n                _request_timeout=None):\n        \"\"\"Perform requests.\n    \n        :param method: http request method\n        :param url: http request url\n        :param query_params: query parameters in the url\n        :param headers: http request headers\n        :param body: request json body, for `application/json`\n        :param post_params: request post parameters,\n                            `application/x-www-form-urlencoded`\n                            and `multipart/form-data`\n        :param _preload_content: if False, the urllib3.HTTPResponse object will\n                                 be returned without reading/decoding response\n                                 data. Default is True.\n        :param _request_timeout: timeout setting for this request. If one\n                                 number provided, it will be total request\n                                 timeout. It can also be a pair (tuple) of\n                                 (connection, read) timeouts.\n        \"\"\"\n        method = method.upper()\n        assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',\n                          'PATCH', 'OPTIONS']\n    \n        if post_params and body:\n            raise ApiValueError(\n                \"body parameter cannot be used with post_params parameter.\"\n            )\n    \n        post_params = post_params or {}\n        headers = headers or {}\n    \n        timeout = None\n        if _request_timeout:\n            if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)):  # noqa: E501,F821\n                timeout = urllib3.Timeout(total=_request_timeout)\n            elif (isinstance(_request_timeout, tuple) and\n                  len(_request_timeout) == 2):\n                timeout = urllib3.Timeout(\n                    connect=_request_timeout[0], read=_request_timeout[1])\n    \n        if 'Content-Type' not in headers:\n            headers['Content-Type'] = 'application/json'\n    \n        try:\n            # For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE`\n            if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:\n                if query_params:\n                    url += '?' + urlencode(query_params)\n                if (re.search('json', headers['Content-Type'], re.IGNORECASE) or\n                        headers['Content-Type'] == 'application/apply-patch+yaml'):\n                    if headers['Content-Type'] == 'application/json-patch+json':\n                        if not isinstance(body, list):\n                            headers['Content-Type'] = \\\n                                'application/strategic-merge-patch+json'\n                    request_body = None\n                    if body is not None:\n                        request_body = json.dumps(body)\n                    r = self.pool_manager.request(\n                        method, url,\n                        body=request_body,\n                        preload_content=_preload_content,\n                        timeout=timeout,\n                        headers=headers)\n                elif headers['Content-Type'] == 'application/x-www-form-urlencoded':  # noqa: E501\n                    r = self.pool_manager.request(\n                        method, url,\n                        fields=post_params,\n                        encode_multipart=False,\n                        preload_content=_preload_content,\n                        timeout=timeout,\n                        headers=headers)\n                elif headers['Content-Type'] == 'multipart/form-data':\n                    # must del headers['Content-Type'], or the correct\n                    # Content-Type which generated by urllib3 will be\n                    # overwritten.\n                    del headers['Content-Type']\n                    r = self.pool_manager.request(\n                        method, url,\n                        fields=post_params,\n                        encode_multipart=True,\n                        preload_content=_preload_content,\n                        timeout=timeout,\n                        headers=headers)\n                # Pass a `string` parameter directly in the body to support\n                # other content types than Json when `body` argument is\n                # provided in serialized form\n                elif isinstance(body, str) or isinstance(body, bytes):\n                    request_body = body\n                    r = self.pool_manager.request(\n                        method, url,\n                        body=request_body,\n                        preload_content=_preload_content,\n                        timeout=timeout,\n                        headers=headers)\n                else:\n                    # Cannot generate the request from given parameters\n                    msg = \"\"\"Cannot prepare a request message for provided\n                             arguments. Please check that your arguments match\n                             declared content type.\"\"\"\n                    raise ApiException(status=0, reason=msg)\n            # For `GET`, `HEAD`\n            else:\n>               r = self.pool_manager.request(method, url,\n                                              fields=query_params,\n                                              preload_content=_preload_content,\n                                              timeout=timeout,\n                                              headers=headers)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:217: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.poolmanager.PoolManager object at 0x7fcebc031410>\nmethod = 'GET'\nurl = 'https://a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/kserve-ci-e2e-test/pods'\nbody = None\nfields = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\njson = None, urlopen_kw = {'preload_content': True, 'timeout': None}\n\n    def request(\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        fields: _TYPE_FIELDS | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        json: typing.Any | None = None,\n        **urlopen_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Make a request using :meth:`urlopen` with the appropriate encoding of\n        ``fields`` based on the ``method`` used.\n    \n        This is a convenience method that requires the least amount of manual\n        effort. It can be used in most situations, while still having the\n        option to drop down to more specific methods when necessary, such as\n        :meth:`request_encode_url`, :meth:`request_encode_body`,\n        or even the lowest level :meth:`urlopen`.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param fields:\n            Data to encode and send in the URL or request body, depending on ``method``.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param json:\n            Data to encode and send as JSON with UTF-encoded in the request body.\n            The ``\"Content-Type\"`` header will be set to ``\"application/json\"``\n            unless specified otherwise.\n        \"\"\"\n        method = method.upper()\n    \n        if json is not None and body is not None:\n            raise TypeError(\n                \"request got values for both 'body' and 'json' parameters which are mutually exclusive\"\n            )\n    \n        if json is not None:\n            if headers is None:\n                headers = self.headers\n    \n            if not (\"content-type\" in map(str.lower, headers.keys())):\n                headers = HTTPHeaderDict(headers)\n                headers[\"Content-Type\"] = \"application/json\"\n    \n            body = _json.dumps(json, separators=(\",\", \":\"), ensure_ascii=False).encode(\n                \"utf-8\"\n            )\n    \n        if body is not None:\n            urlopen_kw[\"body\"] = body\n    \n        if method in self._encode_url_methods:\n>           return self.request_encode_url(\n                method,\n                url,\n                fields=fields,  # type: ignore[arg-type]\n                headers=headers,\n                **urlopen_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/_request_methods.py:135: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.poolmanager.PoolManager object at 0x7fcebc031410>\nmethod = 'GET'\nurl = 'https://a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nfields = [('labelSelector', 'app.kubernetes.io/name=prestop-hook-test,kserve.io/component=workload')]\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nurlopen_kw = {'preload_content': True, 'timeout': None}\nextra_kw = {'headers': {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}, 'preload_content': True, 'timeout': None}\n\n    def request_encode_url(\n        self,\n        method: str,\n        url: str,\n        fields: _TYPE_ENCODE_URL_FIELDS | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        **urlopen_kw: str,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Make a request using :meth:`urlopen` with the ``fields`` encoded in\n        the url. This is useful for request methods like GET, HEAD, DELETE, etc.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param fields:\n            Data to encode and send in the URL.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n        \"\"\"\n        if headers is None:\n            headers = self.headers\n    \n        extra_kw: dict[str, typing.Any] = {\"headers\": headers}\n        extra_kw.update(urlopen_kw)\n    \n        if fields:\n            url += \"?\" + urlencode(fields)\n    \n>       return self.urlopen(method, url, **extra_kw)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/_request_methods.py:182: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.poolmanager.PoolManager object at 0x7fcebc031410>\nmethod = 'GET'\nurl = 'https://a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nredirect = True\nkw = {'assert_same_host': False, 'headers': {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}, 'preload_content': True, 'redirect': False, ...}\nu = Url(scheme='https', auth=None, host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', p..., query='labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload', fragment=None)\nconn = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\n\n    def urlopen(  # type: ignore[override]\n        self, method: str, url: str, redirect: bool = True, **kw: typing.Any\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Same as :meth:`urllib3.HTTPConnectionPool.urlopen`\n        with custom cross-host redirect logic and only sends the request-uri\n        portion of the ``url``.\n    \n        The given ``url`` parameter must be absolute, such that an appropriate\n        :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.\n        \"\"\"\n        u = parse_url(url)\n    \n        if u.scheme is None:\n            warnings.warn(\n                \"URLs without a scheme (ie 'https://') are deprecated and will raise an error \"\n                \"in a future version of urllib3. To avoid this DeprecationWarning ensure all URLs \"\n                \"start with 'https://' or 'http://'. Read more in this issue: \"\n                \"https://github.com/urllib3/urllib3/issues/2920\",\n                category=DeprecationWarning,\n                stacklevel=2,\n            )\n    \n        conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)\n    \n        kw[\"assert_same_host\"] = False\n        kw[\"redirect\"] = False\n    \n        if \"headers\" not in kw:\n            kw[\"headers\"] = self.headers\n    \n        if self._proxy_requires_url_absolute_form(u):\n            response = conn.urlopen(method, url, **kw)\n        else:\n>           response = conn.urlopen(method, u.request_uri, **kw)\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/poolmanager.py:457: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=2, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False, timeout = None, pool_timeout = None\nrelease_conn = True, chunked = False, body_pos = None, preload_content = True\ndecode_content = True, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/api/v1/namespaces/kserve-ci-e2e-test/pods', query='labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload', fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ConnectionResetError(104, 'Connection reset by peer'), clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=1, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False, timeout = None, pool_timeout = None\nrelease_conn = True, chunked = False, body_pos = None, preload_content = True\ndecode_content = True, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/api/v1/namespaces/kserve-ci-e2e-test/pods', query='labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload', fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = ConnectTimeoutError(<HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.c...on to a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com timed out. (connect timeout=None)')\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False, timeout = None, pool_timeout = None\nrelease_conn = True, chunked = False, body_pos = None, preload_content = True\ndecode_content = True, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/api/v1/namespaces/kserve-ci-e2e-test/pods', query='labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload', fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False\nerr = NameResolutionError(\"HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.c...a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com' ([Errno -2] Name or service not known)\")\nclean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n            retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n            retries.sleep()\n    \n            # Keep track of the error for the retry warning.\n            err = e\n    \n        finally:\n            if not clean_exit:\n                # We hit some kind of exception, handled or otherwise. We need\n                # to throw the connection away unless explicitly told not to.\n                # Close the connection, set the variable to None, and make sure\n                # we put the None back in the pool to avoid leaking it.\n                if conn:\n                    conn.close()\n                    conn = None\n                release_this_conn = True\n    \n            if release_this_conn:\n                # Put the connection back to be reused. If the connection is\n                # expired then it will be None, which will get replaced with a\n                # fresh connection during _get_conn.\n                self._put_conn(conn)\n    \n        if not conn:\n            # Try again\n            log.warning(\n                \"Retrying (%r) after connection broken by '%r': %s\", retries, err, url\n            )\n>           return self.urlopen(\n                method,\n                url,\n                body,\n                headers,\n                retries,\n                redirect,\n                assert_same_host,\n                timeout=timeout,\n                pool_timeout=pool_timeout,\n                release_conn=release_conn,\n                chunked=chunked,\n                body_pos=body_pos,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:871: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nbody = None\nheaders = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}\nretries = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nredirect = False, assert_same_host = False, timeout = None, pool_timeout = None\nrelease_conn = True, chunked = False, body_pos = None, preload_content = True\ndecode_content = True, response_kw = {}\nparsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/api/v1/namespaces/kserve-ci-e2e-test/pods', query='labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload', fragment=None)\ndestination_scheme = None, conn = None, release_this_conn = True\nhttp_tunnel_required = False, err = None, clean_exit = False\n\n    def urlopen(  # type: ignore[override]\n        self,\n        method: str,\n        url: str,\n        body: _TYPE_BODY | None = None,\n        headers: typing.Mapping[str, str] | None = None,\n        retries: Retry | bool | int | None = None,\n        redirect: bool = True,\n        assert_same_host: bool = True,\n        timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,\n        pool_timeout: int | None = None,\n        release_conn: bool | None = None,\n        chunked: bool = False,\n        body_pos: _TYPE_BODY_POSITION | None = None,\n        preload_content: bool = True,\n        decode_content: bool = True,\n        **response_kw: typing.Any,\n    ) -> BaseHTTPResponse:\n        \"\"\"\n        Get a connection from the pool and perform an HTTP request. This is the\n        lowest level call for making a request, so you'll need to specify all\n        the raw details.\n    \n        .. note::\n    \n           More commonly, it's appropriate to use a convenience method\n           such as :meth:`request`.\n    \n        .. note::\n    \n           `release_conn` will only behave as expected if\n           `preload_content=False` because we want to make\n           `preload_content=False` the default behaviour someday soon without\n           breaking backwards compatibility.\n    \n        :param method:\n            HTTP request method (such as GET, POST, PUT, etc.)\n    \n        :param url:\n            The URL to perform the request on.\n    \n        :param body:\n            Data to send in the request body, either :class:`str`, :class:`bytes`,\n            an iterable of :class:`str`/:class:`bytes`, or a file-like object.\n    \n        :param headers:\n            Dictionary of custom headers to send, such as User-Agent,\n            If-None-Match, etc. If None, pool headers are used. If provided,\n            these headers completely replace any pool-specific headers.\n    \n        :param retries:\n            Configure the number of retries to allow before raising a\n            :class:`~urllib3.exceptions.MaxRetryError` exception.\n    \n            If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a\n            :class:`~urllib3.util.retry.Retry` object for fine-grained control\n            over different types of retries.\n            Pass an integer number to retry connection errors that many times,\n            but no other types of errors. Pass zero to never retry.\n    \n            If ``False``, then retries are disabled and any exception is raised\n            immediately. Also, instead of raising a MaxRetryError on redirects,\n            the redirect response will be returned.\n    \n        :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.\n    \n        :param redirect:\n            If True, automatically handle redirects (status codes 301, 302,\n            303, 307, 308). Each redirect counts as a retry. Disabling retries\n            will disable redirect, too.\n    \n        :param assert_same_host:\n            If ``True``, will make sure that the host of the pool requests is\n            consistent else will raise HostChangedError. When ``False``, you can\n            use the pool on an HTTP proxy and request foreign hosts.\n    \n        :param timeout:\n            If specified, overrides the default timeout for this one\n            request. It may be a float (in seconds) or an instance of\n            :class:`urllib3.util.Timeout`.\n    \n        :param pool_timeout:\n            If set and the pool is set to block=True, then this method will\n            block for ``pool_timeout`` seconds and raise EmptyPoolError if no\n            connection is available within the time period.\n    \n        :param bool preload_content:\n            If True, the response's body will be preloaded into memory.\n    \n        :param bool decode_content:\n            If True, will attempt to decode the body based on the\n            'content-encoding' header.\n    \n        :param release_conn:\n            If False, then the urlopen call will not release the connection\n            back into the pool once a response is received (but will release if\n            you read the entire contents of the response such as when\n            `preload_content=True`). This is useful if you're not preloading\n            the response's content immediately. You will need to call\n            ``r.release_conn()`` on the response ``r`` to return the connection\n            back into the pool. If None, it takes the value of ``preload_content``\n            which defaults to ``True``.\n    \n        :param bool chunked:\n            If True, urllib3 will send the body using chunked transfer\n            encoding. Otherwise, urllib3 will send the body using the standard\n            content-length form. Defaults to False.\n    \n        :param int body_pos:\n            Position to seek to in file-like body in the event of a retry or\n            redirect. Typically this won't need to be set because urllib3 will\n            auto-populate the value when needed.\n        \"\"\"\n        parsed_url = parse_url(url)\n        destination_scheme = parsed_url.scheme\n    \n        if headers is None:\n            headers = self.headers\n    \n        if not isinstance(retries, Retry):\n            retries = Retry.from_int(retries, redirect=redirect, default=self.retries)\n    \n        if release_conn is None:\n            release_conn = preload_content\n    \n        # Check host\n        if assert_same_host and not self.is_same_host(url):\n            raise HostChangedError(self, url, retries)\n    \n        # Ensure that the URL we're connecting to is properly encoded\n        if url.startswith(\"/\"):\n            url = to_str(_encode_target(url))\n        else:\n            url = to_str(parsed_url.url)\n    \n        conn = None\n    \n        # Track whether `conn` needs to be released before\n        # returning/raising/recursing. Update this variable if necessary, and\n        # leave `release_conn` constant throughout the function. That way, if\n        # the function recurses, the original value of `release_conn` will be\n        # passed down into the recursive call, and its value will be respected.\n        #\n        # See issue #651 [1] for details.\n        #\n        # [1] <https://github.com/urllib3/urllib3/issues/651>\n        release_this_conn = release_conn\n    \n        http_tunnel_required = connection_requires_http_tunnel(\n            self.proxy, self.proxy_config, destination_scheme\n        )\n    \n        # Merge the proxy headers. Only done when not using HTTP CONNECT. We\n        # have to copy the headers dict so we can safely change it without those\n        # changes being reflected in anyone else's copy.\n        if not http_tunnel_required:\n            headers = headers.copy()  # type: ignore[attr-defined]\n            headers.update(self.proxy_headers)  # type: ignore[union-attr]\n    \n        # Must keep the exception bound to a separate variable or else Python 3\n        # complains about UnboundLocalError.\n        err = None\n    \n        # Keep track of whether we cleanly exited the except block. This\n        # ensures we do proper cleanup in finally.\n        clean_exit = False\n    \n        # Rewind body position, if needed. Record current position\n        # for future rewinds in the event of a redirect/retry.\n        body_pos = set_file_position(body, body_pos)\n    \n        try:\n            # Request a connection from the queue.\n            timeout_obj = self._get_timeout(timeout)\n            conn = self._get_conn(timeout=pool_timeout)\n    \n            conn.timeout = timeout_obj.connect_timeout  # type: ignore[assignment]\n    \n            # Is this a closed/new connection that requires CONNECT tunnelling?\n            if self.proxy is not None and http_tunnel_required and conn.is_closed:\n                try:\n                    self._prepare_proxy(conn)\n                except (BaseSSLError, OSError, SocketTimeout) as e:\n                    self._raise_timeout(\n                        err=e, url=self.proxy.url, timeout_value=conn.timeout\n                    )\n                    raise\n    \n            # If we're going to release the connection in ``finally:``, then\n            # the response doesn't need to know about the connection. Otherwise\n            # it will also try to release it and we'll have a double-release\n            # mess.\n            response_conn = conn if not release_conn else None\n    \n            # Make the request on the HTTPConnection object\n            response = self._make_request(\n                conn,\n                method,\n                url,\n                timeout=timeout_obj,\n                body=body,\n                headers=headers,\n                chunked=chunked,\n                retries=retries,\n                response_conn=response_conn,\n                preload_content=preload_content,\n                decode_content=decode_content,\n                **response_kw,\n            )\n    \n            # Everything went great!\n            clean_exit = True\n    \n        except EmptyPoolError:\n            # Didn't get a connection from the pool, no need to clean up\n            clean_exit = True\n            release_this_conn = False\n            raise\n    \n        except (\n            TimeoutError,\n            HTTPException,\n            OSError,\n            ProtocolError,\n            BaseSSLError,\n            SSLError,\n            CertificateError,\n            ProxyError,\n        ) as e:\n            # Discard the connection for these exceptions. It will be\n            # replaced during the next _get_conn() call.\n            clean_exit = False\n            new_e: Exception = e\n            if isinstance(e, (BaseSSLError, CertificateError)):\n                new_e = SSLError(e)\n            if isinstance(\n                new_e,\n                (\n                    OSError,\n                    NewConnectionError,\n                    TimeoutError,\n                    SSLError,\n                    HTTPException,\n                ),\n            ) and (conn and conn.proxy and not conn.has_connected_to_proxy):\n                new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)\n            elif isinstance(new_e, (OSError, HTTPException)):\n                new_e = ProtocolError(\"Connection aborted.\", new_e)\n    \n>           retries = retries.increment(\n                method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]\n            )\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:841: \n_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ \n\nself = Retry(total=0, connect=None, read=None, redirect=None, status=None)\nmethod = 'GET'\nurl = '/api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload'\nresponse = None\nerror = NameResolutionError(\"HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.c...a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com' ([Errno -2] Name or service not known)\")\n_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fcebc0338d0>\n_stacktrace = <traceback object at 0x7fcebc078780>\n\n    def increment(\n        self,\n        method: str | None = None,\n        url: str | None = None,\n        response: BaseHTTPResponse | None = None,\n        error: Exception | None = None,\n        _pool: ConnectionPool | None = None,\n        _stacktrace: TracebackType | None = None,\n    ) -> Self:\n        \"\"\"Return a new Retry object with incremented retry counters.\n    \n        :param response: A response object, or None, if the server did not\n            return a response.\n        :type response: :class:`~urllib3.response.BaseHTTPResponse`\n        :param Exception error: An error encountered during the request, or\n            None if the response was received successfully.\n    \n        :return: A new ``Retry`` object.\n        \"\"\"\n        if self.total is False and error:\n            # Disabled, indicate to re-raise the error.\n            raise reraise(type(error), error, _stacktrace)\n    \n        total = self.total\n        if total is not None:\n            total -= 1\n    \n        connect = self.connect\n        read = self.read\n        redirect = self.redirect\n        status_count = self.status\n        other = self.other\n        cause = \"unknown\"\n        status = None\n        redirect_location = None\n    \n        if error and self._is_connection_error(error):\n            # Connect retry?\n            if connect is False:\n                raise reraise(type(error), error, _stacktrace)\n            elif connect is not None:\n                connect -= 1\n    \n        elif error and self._is_read_error(error):\n            # Read retry?\n            if read is False or method is None or not self._is_method_retryable(method):\n                raise reraise(type(error), error, _stacktrace)\n            elif read is not None:\n                read -= 1\n    \n        elif error:\n            # Other retry?\n            if other is not None:\n                other -= 1\n    \n        elif response and response.get_redirect_location():\n            # Redirect retry?\n            if redirect is not None:\n                redirect -= 1\n            cause = \"too many redirects\"\n            response_redirect_location = response.get_redirect_location()\n            if response_redirect_location:\n                redirect_location = response_redirect_location\n            status = response.status\n    \n        else:\n            # Incrementing because of a server error like a 500 in\n            # status_forcelist and the given method is in the allowed_methods\n            cause = ResponseError.GENERIC_ERROR\n            if response and response.status:\n                if status_count is not None:\n                    status_count -= 1\n                cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)\n                status = response.status\n    \n        history = self.history + (\n            RequestHistory(method, url, error, status, redirect_location),\n        )\n    \n        new_retry = self.new(\n            total=total,\n            connect=connect,\n            read=read,\n            redirect=redirect,\n            status=status_count,\n            other=other,\n            history=history,\n        )\n    \n        if new_retry.is_exhausted():\n            reason = error or ResponseError(cause)\n>           raise MaxRetryError(_pool, url, reason) from reason  # type: ignore[arg-type]\nE           urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443): Max retries exceeded with url: /api/v1/namespaces/kserve-ci-e2e-test/pods?labelSelector=app.kubernetes.io%2Fname%3Dprestop-hook-test%2Ckserve.io%2Fcomponent%3Dworkload (Caused by NameResolutionError(\"HTTPSConnection(host='a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com', port=6443): Failed to resolve 'a2013db6449ea489fa428253bef27c5d-793ec07fd3cc7778.elb.us-east-1.amazonaws.com' ([Errno -2] Name or service not known)\"))\n\n../../python/kserve/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:519: MaxRetryError"}, "teardown": {"duration": 0.0006482999997388106, "outcome": "passed", "longrepr": "[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python"}}], "warnings": [{"message": "The event_loop fixture provided by pytest-asyncio has been redefined in\n/workspace/source/test/e2e/conftest.py:43\nReplacing the event_loop fixture with a custom implementation is deprecated\nand will lead to errors in the future.\nIf you want to request an asyncio event loop with a scope other than function\nscope, use the \"scope\" argument to the asyncio mark when marking the tests.\nIf you want to return different types of event loops, use the event_loop_policy\nfixture.\n", "category": "DeprecationWarning", "when": "runtest", "filename": "/workspace/source/python/kserve/.venv/lib64/python3.11/site-packages/pytest_asyncio/plugin.py", "lineno": 761}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-with-precise-prefix-cache-inline-config-workload-llmd-simulator-kvcache]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-llmd-simulator0]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-llmd-simulator1]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-llmd-simulator2]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The event_loop fixture provided by pytest-asyncio has been redefined in\n/workspace/source/test/e2e/conftest.py:43\nReplacing the event_loop fixture with a custom implementation is deprecated\nand will lead to errors in the future.\nIf you want to request an asyncio event loop with a scope other than function\nscope, use the \"scope\" argument to the asyncio mark when marking the tests.\nIf you want to return different types of event loops, use the event_loop_policy\nfixture.\n", "category": "DeprecationWarning", "when": "runtest", "filename": "/workspace/source/python/kserve/.venv/lib64/python3.11/site-packages/pytest_asyncio/plugin.py", "lineno": 761}, {"message": "The test <Function test_llm_inference_service[router-with-gateway-ref-router-with-managed-route-model-fb-opt-125m-workload-llmd-simulator]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf0]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-single-cpu-model-fb-opt-125m-with-lora-hf1]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-custom-route-timeout-scheduler-managed-workload-single-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-single-cpu-model-pvc]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-with-refs-scheduler-managed-workload-single-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-pd-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-custom-route-timeout-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-pd-cpu-model-pvc]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-with-refs-pd-scheduler-managed-workload-pd-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-no-scheduler-workload-single-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-simulated-dp-ep-cpu-model-pvc]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-simulated-dp-ep-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-with-inline-config-workload-llmd-simulator]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-workload-llmd-simulator-model-qwen2.5-0.5b]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_stop_feature[router-managed-workload-single-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service_stop.py", "lineno": 40}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-with-configmap-ref-workload-llmd-simulator]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-with-replicas-workload-llmd-simulator]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-with-custom-template-workload-llmd-simulator]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-v06-pd-config-migration-workload-llmd-simulator-pd]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}, {"message": "The test <Function test_llm_tls_resources[router-managed-workload-single-cpu-model-fb-opt-125m]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_tls.py", "lineno": 93}, {"message": "The test <Function test_llm_inference_service[router-managed-scheduler-v06-nonzero-threshold-migration-workload-llmd-simulator-pd]> is marked with '@pytest.mark.asyncio' but it is not an async function. Please remove the asyncio mark. If the test is not marked explicitly, check for global marks applied via 'pytestmark'.", "category": "PytestWarning", "when": "runtest", "filename": "llmisvc/test_llm_inference_service.py", "lineno": 245}]}