cls = <class 'ssl.SSLSocket'> sock = <socket.socket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6> server_side = False, do_handshake_on_connect = True, suppress_ragged_eofs = True server_hostname = 'maas.apps.1fd4c4aa-417e-4700-817c-4a8707020974.prod.konfluxeaas.com' context = <ssl.SSLContext object at 0x7f14c1019240>, session = None @classmethod def _create(cls, sock, server_side=False, do_handshake_on_connect=True, suppress_ragged_eofs=True, server_hostname=None, context=None, session=None): if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM: raise NotImplementedError("only stream sockets are supported") if server_side: if server_hostname: raise ValueError("server_hostname can only be specified " "in client mode") if session is not None: raise ValueError("session can only be specified in " "client mode") if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) sock_timeout = sock.gettimeout() sock.detach() self._context = context self._session = session self._closed = False self._sslobj = None self.server_side = server_side self.server_hostname = context._encode_hostname(server_hostname) self.do_handshake_on_connect = do_handshake_on_connect self.suppress_ragged_eofs = suppress_ragged_eofs # See if we are connected try: > self.getpeername() E OSError: [Errno 107] Transport endpoint is not connected /usr/lib64/python3.9/ssl.py:1022: OSError During handling of the above exception, another exception occurred: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f14c0ffc490> method = 'DELETE' url = '/maas-api/v1/api-keys/e57b437a-74b6-44de-b31d-8e6d2ce70304', body = None headers = {'User-Agent': 'python-requests/2.32.5', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...gtNhhUXMvJ2pY7obGVNJ0N2pZwVsBDBFplM047MTKejCpBVaFueSau-M0A', 'Content-Type': 'application/json', 'Content-Length': '0'} retries = Retry(total=0, connect=None, read=False, redirect=None, status=None) redirect = False, assert_same_host = False timeout = Timeout(connect=30, read=30, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/maas-api/v1/api-keys/e57b437a-74b6-44de-b31d-8e6d2ce70304', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:464: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connection.py:796: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( test/e2e/.venv/lib64/python3.9/site-packages/urllib3/util/ssl_.py:483: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/util/ssl_.py:527: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) /usr/lib64/python3.9/ssl.py:501: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.9/ssl.py:1034: in _create notconn_pre_handshake_data = self.recv(1) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket fd=5, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6, laddr=('10.128.136.145', 50194)> buflen = 1, flags = 0 def recv(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: if flags != 0: raise ValueError( "non-zero flags not allowed in calls to recv() on %s" % self.__class__) return self.read(buflen) else: > return super().recv(buflen, flags) E ConnectionResetError: [Errno 104] Connection reset by peer /usr/lib64/python3.9/ssl.py:1264: ConnectionResetError During handling of the above exception, another exception occurred: self = <requests.adapters.HTTPAdapter object at 0x7f14c0ffc910> request = <PreparedRequest [DELETE]>, stream = False timeout = Timeout(connect=30, read=30, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) test/e2e/.venv/lib64/python3.9/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( test/e2e/.venv/lib64/python3.9/site-packages/urllib3/util/retry.py:490: in increment raise reraise(type(error), error, _stacktrace) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/util/util.py:38: in reraise raise value.with_traceback(tb) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:787: in urlopen response = self._make_request( test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:464: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connection.py:796: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( test/e2e/.venv/lib64/python3.9/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( test/e2e/.venv/lib64/python3.9/site-packages/urllib3/util/ssl_.py:483: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) test/e2e/.venv/lib64/python3.9/site-packages/urllib3/util/ssl_.py:527: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) /usr/lib64/python3.9/ssl.py:501: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.9/ssl.py:1034: in _create notconn_pre_handshake_data = self.recv(1) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket fd=5, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6, laddr=('10.128.136.145', 50194)> buflen = 1, flags = 0 def recv(self, buflen=1024, flags=0): self._checkClosed() if self._sslobj is not None: if flags != 0: raise ValueError( "non-zero flags not allowed in calls to recv() on %s" % self.__class__) return self.read(buflen) else: > return super().recv(buflen, flags) E urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')) /usr/lib64/python3.9/ssl.py:1264: ProtocolError During handling of the above exception, another exception occurred: self = <test_api_keys.TestAPIKeyRevocationE2E object at 0x7f14c1033b50> api_keys_base_url = 'https://maas.apps.1fd4c4aa-417e-4700-817c-4a8707020974.prod.konfluxeaas.com/maas-api/v1/api-keys' model_completions_url = 'https://maas.apps.1fd4c4aa-417e-4700-817c-4a8707020974.prod.konfluxeaas.com/llm/facebook-opt-125m-simulated/v1/completions' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Ik10ZHN2eXN0WWs5UDVEWl90SGlLYkhmTlc1OU9wckp3SDdFaEd5bnNOMFkifQ.e...W8r2896PPkQ43s3MWkHrWhIgtNhhUXMvJ2pY7obGVNJ0N2pZwVsBDBFplM047MTKejCpBVaFueSau-M0A', 'Content-Type': 'application/json'} inference_model_name = 'facebook/opt-125m' def test_revoke_keys_rejected_at_gateway( self, api_keys_base_url: str, model_completions_url: str, headers: dict, inference_model_name: str, ): """After individually revoking keys, they should be rejected at the gateway.""" designated = SIMULATOR_SUBSCRIPTION # Create 3 keys, capturing plaintext and IDs keys = [] for i in range(3): r = requests.post( api_keys_base_url, headers=headers, json={"name": f"test-revoke-gw-{i}", "subscription": designated}, timeout=30, verify=TLS_VERIFY, ) assert r.status_code in (200, 201), f"Failed to create key {i}: {r.text}" keys.append({"id": r.json()["id"], "key": r.json()["key"]}) # Smoke-test: verify at least one key works before revocation r_smoke = requests.post( model_completions_url, headers={"Authorization": f"Bearer {keys[0]['key']}", "Content-Type": "application/json"}, json={"model": inference_model_name, "prompt": "Test", "max_tokens": 5}, timeout=60, verify=TLS_VERIFY, ) assert r_smoke.status_code == 200, ( f"Key should work before revoke, got {r_smoke.status_code}: {r_smoke.text}" ) # Individually revoke all 3 keys (not bulk-revoke, to avoid nuking session key) for k in keys: > r = requests.delete(f"{api_keys_base_url}/{k['id']}", headers=headers, timeout=30, verify=TLS_VERIFY) test/e2e/tests/test_api_keys.py:857: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/requests/api.py:157: in delete return request("delete", url, **kwargs) test/e2e/.venv/lib64/python3.9/site-packages/requests/api.py:59: in request return session.request(method=method, url=url, **kwargs) test/e2e/.venv/lib64/python3.9/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) test/e2e/.venv/lib64/python3.9/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <requests.adapters.HTTPAdapter object at 0x7f14c0ffc910> request = <PreparedRequest [DELETE]>, stream = False timeout = Timeout(connect=30, read=30, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: > raise ConnectionError(err, request=request) E requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')) test/e2e/.venv/lib64/python3.9/site-packages/requests/adapters.py:659: ConnectionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_external_oidc.py:420: OIDC_USERNAME_NO_ACCESS and OIDC_PASSWORD_NO_ACCESS not configured