self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowExceptionself = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) timeout = Timeout(connect=3, read=3, total=None), chunked = False response_conn = <urllib3.connection.HTTPSConnection object at 0x7f1824db1b80> preload_content = False, decode_content = False, enforce_content_length = True def _make_request( self, conn: BaseHTTPConnection, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | None = None, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, chunked: bool = False, response_conn: BaseHTTPConnection | None = None, preload_content: bool = True, decode_content: bool = True, enforce_content_length: bool = True, ) -> BaseHTTPResponse: """ Perform a request on a given urllib connection object taken from our pool. :param conn: a connection from one of our connection pools :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. Pass ``None`` to retry until you receive a response. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param response_conn: Set this to ``None`` if you will handle releasing the connection or set the connection to have the response release it. :param preload_content: If True, the response's body will be preloaded during construction. :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param enforce_content_length: Enforce content length checking. Body returned by server must match value of Content-Length header, if present. Otherwise, raise error. """ self.num_requests += 1 timeout_obj = self._get_timeout(timeout) timeout_obj.start_connect() conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout) try: # Trigger any extra validation we need to do. try: > self._validate_conn(conn) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:464: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:1093: in _validate_conn conn.connect() .venv/lib64/python3.12/site-packages/urllib3/connection.py:741: in connect sock_and_verified = _ssl_wrap_socket_and_match_hostname( .venv/lib64/python3.12/site-packages/urllib3/connection.py:920: in _ssl_wrap_socket_and_match_hostname ssl_sock = ssl_wrap_socket( .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:460: in ssl_wrap_socket ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/urllib3/util/ssl_.py:504: in _ssl_wrap_socket_impl return ssl_context.wrap_socket(sock, server_hostname=server_hostname) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ /usr/lib64/python3.12/ssl.py:455: in wrap_socket return self.sslsocket_class._create( /usr/lib64/python3.12/ssl.py:1041: in _create self.do_handshake() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False @_sslcopydoc def do_handshake(self, block=False): self._check_connected() timeout = self.gettimeout() try: if timeout == 0.0 and block: self.settimeout(None) > self._sslobj.do_handshake() E TimeoutError: _ssl.c:981: The handshake operation timed out /usr/lib64/python3.12/ssl.py:1319: TimeoutError The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> method = 'GET', url = '/mlflow/api/3.0/mlflow/server-info', body = None headers = {'User-Agent': 'mlflow-python-client/3.12.0', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep...dsJM10xS3xd1yMsyftLPYHVnov1kvmHKDPZTdc3bNS-O0cqldcd0wDH5zZ9D9Z3_gtoDWoGxG4EQ0BHFO-MIP8T3fgQVjvI-8E89WCCS8obuvrR4x3shA'} retries = Retry(total=0, connect=0, read=0, redirect=0, status=0) redirect = False, assert_same_host = False timeout = Timeout(connect=3, read=3, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} parsed_url = Url(scheme=None, auth=None, host=None, port=None, path='/mlflow/api/3.0/mlflow/server-info', query=None, fragment=None) destination_scheme = None, conn = None, release_this_conn = True http_tunnel_required = False, err = None, clean_exit = False def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ parsed_url = parse_url(url) destination_scheme = parsed_url.scheme if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): url = to_str(_encode_target(url)) else: url = to_str(parsed_url.url) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) try: # Request a connection from the queue. timeout_obj = self._get_timeout(timeout) conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:787: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:488: in _make_request raise new_e .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:466: in _make_request self._raise_timeout(err=e, url=url, timeout_value=conn.timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> err = TimeoutError('_ssl.c:981: The handshake operation timed out') url = '/mlflow/api/3.0/mlflow/server-info', timeout_value = 3 def _raise_timeout( self, err: BaseSSLError | OSError | SocketTimeout, url: str, timeout_value: _TYPE_TIMEOUT | None, ) -> None: """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): > raise ReadTimeoutError( self, url, f"Read timed out. (read timeout={timeout_value})" ) from err E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3) .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError The above exception was the direct cause of the following exception: self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) .venv/lib64/python3.12/site-packages/requests/adapters.py:644: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/urllib3/connectionpool.py:841: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=0, read=0, redirect=0, status=0), method = 'GET' url = '/mlflow/api/3.0/mlflow/server-info', response = None error = ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7f1824be1ee0> _stacktrace = <traceback object at 0x7f18235eb340> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/urllib3/util/retry.py:519: MaxRetryError During handling of the above exception, another exception occurred: host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: > return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/utils/request_utils.py:282: in _get_http_response_with_retries return session.request(method, url, allow_redirects=allow_redirects, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:589: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/requests/sessions.py:703: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.utils.request_utils.TCPKeepAliveHTTPAdapter object at 0x7f1824dbbef0> request = <PreparedRequest [GET]>, stream = False timeout = Timeout(connect=3, read=3, total=None), verify = False, cert = None proxies = OrderedDict() def send( self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None ): """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): pass else: timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, headers=request.headers, redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/requests/adapters.py:677: ConnectionError During handling of the above exception, another exception occurred: self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: > response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:57: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host_creds = <mlflow.utils.rest_utils.MlflowHostCreds object at 0x7f1824be2480> endpoint = '/api/3.0/mlflow/server-info', method = 'GET', max_retries = 0 backoff_factor = 2, backoff_jitter = 1.0, extra_headers = None retry_codes = frozenset({408, 429, 500, 502, 503, 504}), timeout = 3 raise_on_status = False, respect_retry_after_header = True retry_timeout_seconds = None, kwargs = {} cleaned_hostname = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow' url = 'https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info' resolve_request_headers = <function resolve_request_headers at 0x7f18235b2020> workspace = 'workspace1-8csnmcym', traffic_id = None def http_request( host_creds, endpoint, method, max_retries=None, backoff_factor=None, backoff_jitter=None, extra_headers=None, retry_codes=_TRANSIENT_FAILURE_RESPONSE_CODES, timeout=None, raise_on_status=True, respect_retry_after_header=None, retry_timeout_seconds=None, **kwargs, ): """Makes an HTTP request with the specified method to the specified hostname/endpoint. Transient errors such as Rate-limited (429), service unavailable (503) and internal error (500) are retried with an exponential back off with backoff_factor * (1, 2, 4, ... seconds). The function parses the API response (assumed to be JSON) into a Python object and returns it. Args: host_creds: A :py:class:`mlflow.rest_utils.MlflowHostCreds` object containing hostname and optional authentication. endpoint: A string for service endpoint, e.g. "/path/to/object". method: A string indicating the method to use, e.g. "GET", "POST", "PUT". max_retries: Maximum number of retries before throwing an exception. backoff_factor: A time factor for exponential backoff. e.g. value 5 means the HTTP request will be retried with interval 5, 10, 20... seconds. A value of 0 turns off the exponential backoff. backoff_jitter: A random jitter to add to the backoff interval. extra_headers: A dict of HTTP header name-value pairs to be included in the request. retry_codes: A list of HTTP response error codes that qualifies for retry. timeout: Wait for timeout seconds for response from remote server for connect and read request. raise_on_status: Whether to raise an exception, or return a response, if status falls in retry_codes range and retries have been exhausted. respect_retry_after_header: Whether to respect Retry-After header on status codes defined as Retry.RETRY_AFTER_STATUS_CODES or not. retry_timeout_seconds: Timeout for retries. Only effective when using Databricks SDK. kwargs: Additional keyword arguments to pass to `requests.Session.request()` Returns: requests.Response object. """ cleaned_hostname = strip_suffix(host_creds.host, "/") url = f"{cleaned_hostname}{endpoint}" # Set defaults for retry parameters from environment variables if not specified max_retries = MLFLOW_HTTP_REQUEST_MAX_RETRIES.get() if max_retries is None else max_retries backoff_factor = ( MLFLOW_HTTP_REQUEST_BACKOFF_FACTOR.get() if backoff_factor is None else backoff_factor ) backoff_jitter = ( MLFLOW_HTTP_REQUEST_BACKOFF_JITTER.get() if backoff_jitter is None else backoff_jitter ) from mlflow.tracking.request_header.registry import resolve_request_headers headers = dict(**resolve_request_headers()) if extra_headers: headers = dict(**headers, **extra_headers) workspace = get_request_workspace() if workspace and _should_include_workspace_header(endpoint): headers.setdefault(WORKSPACE_HEADER_NAME, workspace) if traffic_id := _MLFLOW_DATABRICKS_TRAFFIC_ID.get(): headers["x-databricks-traffic-id"] = traffic_id if host_creds.workspace_id: headers["x-databricks-org-id"] = host_creds.workspace_id if host_creds.use_databricks_sdk: from databricks.sdk.errors import DatabricksError ws_client = get_workspace_client( host_creds.use_secret_scope_token, host_creds.host, host_creds.token, host_creds.databricks_auth_profile, retry_timeout_seconds=retry_timeout_seconds, timeout=timeout, ) def make_sdk_call(): # Databricks SDK `APIClient.do` API is for making request using # HTTP # https://github.com/databricks/databricks-sdk-py/blob/a714146d9c155dd1e3567475be78623f72028ee0/databricks/sdk/core.py#L134 # suppress the warning due to https://github.com/databricks/databricks-sdk-py/issues/963 with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message=f".*{_DATABRICKS_SDK_RETRY_AFTER_SECS_DEPRECATION_WARNING}.*" ) raw_response = ws_client.api_client.do( method=method, path=endpoint, headers=headers, raw=True, query=kwargs.get("params"), body=kwargs.get("json"), files=kwargs.get("files"), data=kwargs.get("data"), ) return raw_response["contents"]._response try: # We retry the SDK call with exponential backoff because the Databricks SDK default # retry behavior does not handle all transient errors that we want to retry, and it # does not support a customizable retry policy based on HTTP response status codes. # Note that, in uncommon cases (due to the limited set if HTTP status codes and # response strings that Databricks SDK retries on), the SDK may retry internally, # and MLflow may retry on top of that, leading to more retries than specified by # `max_retries`. This is acceptable, given the enforcement of an overall request # timeout via `retry_timeout_seconds`. # # TODO: Update transient error handling defaults in Databricks SDK to match standard # practices (e.g. retrying on 429, 500, 503, etc.), support custom retries in Databricks # SDK, and remove this custom retry wrapper from MLflow return _retry_databricks_sdk_call_with_exponential_backoff( call_func=make_sdk_call, retry_codes=retry_codes, retry_timeout_seconds=( retry_timeout_seconds if retry_timeout_seconds is not None else MLFLOW_DATABRICKS_ENDPOINT_HTTP_RETRY_TIMEOUT.get() ), backoff_factor=backoff_factor, backoff_jitter=backoff_jitter, max_retries=max_retries, ) except DatabricksError as e: response = requests.Response() response.url = url response.status_code = ERROR_CODE_TO_HTTP_STATUS.get(e.error_code, 500) response.reason = str(e) response.encoding = "UTF-8" response._content = json.dumps({ "error_code": e.error_code, "message": str(e), }).encode("UTF-8") return response _validate_max_retries(max_retries) _validate_backoff_factor(backoff_factor) respect_retry_after_header = ( MLFLOW_HTTP_RESPECT_RETRY_AFTER_HEADER.get() if respect_retry_after_header is None else respect_retry_after_header ) timeout = MLFLOW_HTTP_REQUEST_TIMEOUT.get() if timeout is None else timeout auth_str = None if host_creds.username and host_creds.password: basic_auth_str = f"{host_creds.username}:{host_creds.password}".encode() auth_str = "Basic " + base64.standard_b64encode(basic_auth_str).decode("utf-8") elif host_creds.token: auth_str = f"Bearer {host_creds.token}" elif host_creds.client_secret: raise MlflowException( "To use OAuth authentication, set environmental variable " f"'{MLFLOW_ENABLE_DB_SDK.name}' to true", error_code=CUSTOMER_UNAUTHORIZED, ) if auth_str: headers["Authorization"] = auth_str if host_creds.client_cert_path is not None: kwargs["cert"] = host_creds.client_cert_path if host_creds.aws_sigv4: # will overwrite the Authorization header from requests_auth_aws_sigv4 import AWSSigV4 kwargs["auth"] = AWSSigV4("execute-api") elif host_creds.auth: from mlflow.tracking.request_auth.registry import fetch_auth kwargs["auth"] = fetch_auth(host_creds.auth) try: return _get_http_response_with_retries( method, url, max_retries, backoff_factor, backoff_jitter, retry_codes, raise_on_status, headers=headers, verify=host_creds.verify, timeout=timeout, respect_retry_after_header=respect_retry_after_header, **kwargs, ) except requests.exceptions.Timeout as to: raise MlflowException( f"API request to {url} failed with timeout exception {to}." " To increase the timeout, set the environment variable " f"{MLFLOW_HTTP_REQUEST_TIMEOUT!s} to a larger value." ) from to except requests.exceptions.InvalidURL as iu: raise InvalidUrlException(f"Invalid url: {url}") from iu except Exception as e: > raise MlflowException(f"API request to {url} failed with exception {e}") E mlflow.exceptions.MlflowException: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/utils/rest_utils.py:305: MlflowException The above exception was the direct cause of the following exception: setup_clients = (<mlflow.tracking.client.MlflowClient object at 0x7f1824be5790>, <mlflow_tests.manager.namespace.K8Manager object at 0...0>, <mlflow_tests.manager.user.K8UserManager object at 0x7f1825f63080>, ['workspace1-8csnmcym', 'workspace2-8csnmcym']) @pytest.fixture(autouse=True, scope="session") def create_experiments_and_runs(setup_clients): """Create session-scoped test resources for all workspaces. This fixture runs once per test session and creates baseline resources (experiments, runs, registered models) that tests can use for validation and permission checks. Note: This fixture uses the admin_client credentials that were set during setup_clients. The mlflow module will use those credentials since they are set in the environment variables. """ import mlflow logger.info("=" * 80) logger.info("CREATING SESSION-SCOPED TEST RESOURCES") logger.info("=" * 80) admin_client, k8_manager, user_manager, workspaces = setup_clients if is_upgrade_phase(): logger.info("Skipping baseline resource creation for upgrade-only pytest phase") logger.info("=" * 80) return {} resource_map = dict() # Verify admin authentication is properly set logger.debug("Verifying admin authentication credentials are set") if not os.environ.get('MLFLOW_TRACKING_TOKEN'): logger.warning("MLFLOW_TRACKING_TOKEN not set - admin client may not be authenticated") logger.info(f"Creating baseline resources for {len(workspaces)} workspaces") for workspace in workspaces: logger.info(f"Processing workspace: {workspace}") mlflow.set_workspace(workspace) logger.debug(f"Set active workspace to: {workspace}") experiment_resources = {} for slot in ("primary", "secondary"): experiment_name = f"test-experiment-{slot}-{random_gen.randint(1, 10000)}" logger.debug(f"Creating {slot} baseline experiment: {experiment_name}") try: > experiment_id = mlflow.create_experiment(experiment_name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ tests/conftest.py:310: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .venv/lib64/python3.12/site-packages/mlflow/tracking/fluent.py:2346: in create_experiment experiment_id = client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/client.py:2101: in create_experiment return self._tracking_client.create_experiment(name, artifact_location, tags) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/telemetry/track.py:30: in wrapper result = func(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/tracking/_tracking_service/client.py:301: in create_experiment return self.store.create_experiment( .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:294: in create_experiment response_proto = self._call_endpoint(CreateExperiment, req_body) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/tracking/rest_store.py:234: in _call_endpoint self._validate_workspace_support_if_specified() .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:48: in _validate_workspace_support_if_specified if not self.supports_workspaces: ^^^^^^^^^^^^^^^^^^^^^^^^ .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:37: in supports_workspaces supported = self._probe_workspace_support() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <mlflow.store.tracking.rest_store.RestStore object at 0x7f1824be79e0> def _probe_workspace_support(self) -> bool: host_creds = self.get_host_creds() try: response = http_request( host_creds=host_creds, endpoint=self._SERVER_INFO_ENDPOINT, method="GET", timeout=3, max_retries=0, raise_on_status=False, ) except Exception as exc: # pragma: no cover - network errors vary > raise MlflowException( message=f"Failed to query {self._SERVER_INFO_ENDPOINT}: {exc}", error_code=databricks_pb2.INTERNAL_ERROR, ) from exc E mlflow.exceptions.MlflowException: Failed to query /api/3.0/mlflow/server-info: API request to https://rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com/mlflow/api/3.0/mlflow/server-info failed with exception HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Max retries exceeded with url: /mlflow/api/3.0/mlflow/server-info (Caused by ReadTimeoutError("HTTPSConnectionPool(host='rh-ai.apps.78e12fd3-8217-40dd-a024-df66a9130456.prod.konfluxeaas.com', port=443): Read timed out. (read timeout=3)")) .venv/lib64/python3.12/site-packages/mlflow/store/workspace_rest_store_mixin.py:66: MlflowException