api_keys_base_url = 'https://maas.apps.3cc7987f-f9bc-44a3-9e97-14e1a63d9621.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9XZF82eGxKSGtCZDdmV3pzbFdkWXhrNE1kQ2l2TmdSLWlRUkdPR2hXNlEifQ.e...Ueve3eiGxBZ4a9QGe02uKVP-k1aOFCY-prQOqJKBa3f_9poTiC4bxtC0JLyjjCy4DIbkEsO7NtWHF07Qg', 'Content-Type': 'application/json'} @pytest.fixture(scope="session", autouse=True) def _warm_gateway(api_keys_base_url: str, headers: dict): """Wait for gateway AuthPolicy enforcement before running API key tests. MaaSAuthPolicy churn in earlier modules (or this session) can leave maas-gateway-auth reconciling; empty 403s follow until Enforced=True and Envoy loads the config. """ > _wait_for_gateway_auth_enforced() test/e2e/tests/test_api_keys.py:82: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:958: in _wait_for_gateway_auth_enforced cr = _get_cr("authpolicy", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'authpolicy', name = 'maas-gateway-auth', namespace = 'openshift-ingress' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get authpolicy/maas-gateway-auth in namespace 'openshift-ingress': Unable to connect to the server: net/http: TLS handshake timeout test/e2e/tests/test_helper.py:460: RuntimeErrorapi_keys_base_url = 'https://maas.apps.3cc7987f-f9bc-44a3-9e97-14e1a63d9621.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im9XZF82eGxKSGtCZDdmV3pzbFdkWXhrNE1kQ2l2TmdSLWlRUkdPR2hXNlEifQ.e...Ueve3eiGxBZ4a9QGe02uKVP-k1aOFCY-prQOqJKBa3f_9poTiC4bxtC0JLyjjCy4DIbkEsO7NtWHF07Qg', 'Content-Type': 'application/json'} @pytest.fixture(scope="session", autouse=True) def _warm_gateway(api_keys_base_url: str, headers: dict): """Wait for gateway AuthPolicy enforcement before running API key tests. MaaSAuthPolicy churn in earlier modules (or this session) can leave maas-gateway-auth reconciling; empty 403s follow until Enforced=True and Envoy loads the config. """ > _wait_for_gateway_auth_enforced() test/e2e/tests/test_api_keys.py:82: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:958: in _wait_for_gateway_auth_enforced cr = _get_cr("authpolicy", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'authpolicy', name = 'maas-gateway-auth', namespace = 'openshift-ingress' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get authpolicy/maas-gateway-auth in namespace 'openshift-ingress': Unable to connect to the server: net/http: TLS handshake timeout test/e2e/tests/test_helper.py:460: RuntimeErrorself = <test_negative_security.TestAuthPolicyRemoval object at 0x7f85f48f9ee0> @pytest.mark.serial def test_authpolicy_deletion_revokes_access(self): """Create auth policy, delete it, verify legacy per-model AuthPolicy is absent. Uses the unconfigured model to avoid interfering with other tests. In gateway-only mode, the controller should not create per-model Kuadrant AuthPolicies. This verifies the legacy per-model AuthPolicy remains absent before and after MaaSAuthPolicy deletion. This tests the controller's cleanup logic. Gateway enforcement of AuthPolicy is already covered by other tests (e.g. test_wrong_group_gets_403). """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-policy-{suffix}" model_ref = UNCONFIGURED_MODEL_REF kuadrant_auth_name = f"maas-auth-{model_ref}" try: # Create auth policy granting access > _create_test_auth_policy( policy_name, model_ref, groups=["system:authenticated"], ) test/e2e/tests/test_negative_security.py:383: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:680: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-neg-policy-b3142e1...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'error: error when retrieving current configuration of:\nResource: "maas.opendatahub.io/v1alpha1, Resource=maasauthpol...o/v1alpha1/namespaces/models-as-a-service/maasauthpolicies/e2e-neg-policy-b3142e11": net/http: TLS handshake timeout\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f85f48b2df0> @pytest.mark.serial def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated > _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) test/e2e/tests/test_subscription.py:467: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:680: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-rate-limit-test-au...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'error: error when retrieving current configuration of:\nResource: "maas.opendatahub.io/v1alpha1, Resource=maasauthpol.../v1alpha1/namespaces/models-as-a-service/maasauthpolicies/e2e-rate-limit-test-auth": net/http: TLS handshake timeout\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessError During handling of the above exception, another exception occurred: self = <test_subscription.TestSubscriptionEnforcement object at 0x7f85f48b2df0> @pytest.mark.serial def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_for_maas_subscription_phase(subscription_name) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. API key must be minted for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-rate-limit-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Send requests to exhaust the limit rate_limited = False success_count = 0 for i in range(total_requests): r = _inference(api_key, path=model_path, max_tokens=1) request_num = i + 1 log.info(f"Request {request_num}/{total_requests}: {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: rate_limited = True log.info(f"Rate limit exceeded after {success_count} successful requests") # Verify it's a rate limit 429, not a subscription error response_text = r.text.lower() if r.text else "" # Rate limit 429s typically mention "rate", "limit", or "quota" # Subscription 429s mention "subscription" without "rate" is_rate_limit_error = any(keyword in response_text for keyword in ["rate", "limit", "quota", "too many"]) is_subscription_error = "subscription" in response_text and not is_rate_limit_error assert is_rate_limit_error or not is_subscription_error, \ f"Expected rate limit 429, not subscription error. Response: {r.text[:500]}" # Check for Retry-After header (optional but good practice) retry_after = r.headers.get("Retry-After") or r.headers.get("retry-after") if retry_after: log.info(f"Retry-After header present: {retry_after}") break else: # Unexpected status code raise AssertionError(f"Unexpected status {r.status_code} at request {request_num}: {r.text[:200]}") # Brief pause to avoid overwhelming the system, but stay within the window time.sleep(0.1) # Verify we actually exhausted the limit (at least one successful request) assert success_count > 0, \ f"Got 429 on request #{request_num} without any successful requests. " \ f"This indicates a configuration issue, not rate limit exhaustion. Response: {r.text[:500]}" assert rate_limited, \ f"Expected 429 with {token_limit} tokens/{window} limit, " \ f"but got {success_count} successful requests without hitting limit" # Note: Skipping rate limit reset test to keep test fast (<5s) # Reset behavior is tested manually via scripts/test-rate-limit.sh finally: # Clean up in reverse order of creation _delete_cr("maassubscription", subscription_name) _delete_cr("maasauthpolicy", auth_policy_name) > _wait_for_cr_absent("maassubscription", subscription_name) test/e2e/tests/test_subscription.py:551: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:1321: in _wait_for_cr_absent if _get_cr(kind, name, namespace) is None: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-rate-limit-test-subscription' namespace = 'models-as-a-service' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get maassubscription/e2e-rate-limit-test-subscription in namespace 'models-as-a-service': The connection to the server af046925e96a741e98f857a2cbffd881-b2d009efa8be824a.elb.us-east-1.amazonaws.com:6443 was refused - did you specify the right host or port? test/e2e/tests/test_helper.py:460: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f85f48ba070> @pytest.mark.serial def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated > _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) test/e2e/tests/test_subscription.py:588: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:680: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-models-exempt-test...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'error: error when retrieving current configuration of:\nResource: "maas.opendatahub.io/v1alpha1, Resource=maasauthpol...as-a-service/maasauthpolicies/e2e-models-exempt-test-auth": dial tcp 100.58.21.210:6443: connect: connection refused\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessError During handling of the above exception, another exception occurred: self = <test_subscription.TestSubscriptionEnforcement object at 0x7f85f48ba070> @pytest.mark.serial def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90, require_auth_policies=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Exhaust the token limit # With 3 token limit and 5 requests, we're guaranteed to hit the limit # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) max_requests = 5 success_count = 0 rate_limited = False log.info(f"Exhausting token quota: sending up to {max_requests} requests") for i in range(max_requests): r = _inference(api_key, path=model_path) request_num = i + 1 log.info(f"Request {request_num}: status {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: log.info(f"Rate limit hit after {success_count} successful requests") rate_limited = True break else: # Unexpected status during exhaustion log.warning(f"Unexpected status during quota exhaustion: {r.status_code}") # Verify we hit rate limit (otherwise test setup is broken) assert rate_limited, \ f"Expected to hit rate limit within {max_requests} requests with {token_limit} token limit, " \ f"but got {success_count} successful requests without hitting limit" # 5. Verify inference is now blocked with 429 log.info("Verifying inference endpoint is blocked...") r_inference = _inference(api_key, path=model_path) assert r_inference.status_code == 429, \ f"Expected 429 for inference after exhausting tokens, got {r_inference.status_code}. " \ f"Response: {r_inference.text[:500]}" log.info("✓ Inference endpoint correctly blocked with 429") # 6. Verify /v1/models endpoint is still accessible with 200 log.info("Verifying /v1/models endpoint is still accessible...") url = f"{_gateway_url()}{model_path}/v1/models" headers = {"Authorization": f"Bearer {api_key}"} r_models = _request_with_gateway_retry(requests.get, url, headers=headers) assert r_models.status_code == 200, \ f"Expected 200 for /v1/models endpoint even when quota exhausted, got {r_models.status_code}. " \ f"The /v1/models endpoint does not consume tokens and should remain accessible. " \ f"Response: {r_models.text[:500]}" # Verify it returns valid model metadata (sanity check) try: models_data = r_models.json() except (json.JSONDecodeError, ValueError) as e: # Non-JSON response is acceptable for some vLLM versions log.info(f"✓ /v1/models endpoint accessible (200), non-JSON response: {r_models.text[:200]}") else: # JSON response - validate structure assert "data" in models_data or "object" in models_data, \ f"Expected valid models response with 'data' or 'object' field, got: {models_data}" log.info(f"✓ /v1/models endpoint accessible (200) despite exhausted quota. Response keys: {list(models_data.keys())}") finally: # Clean up _delete_cr("maassubscription", subscription_name) _delete_cr("maasauthpolicy", auth_policy_name) > _wait_for_cr_absent("maassubscription", subscription_name) test/e2e/tests/test_subscription.py:679: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:1321: in _wait_for_cr_absent if _get_cr(kind, name, namespace) is None: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-models-exempt-test-subscription' namespace = 'models-as-a-service' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get maassubscription/e2e-models-exempt-test-subscription in namespace 'models-as-a-service': The connection to the server af046925e96a741e98f857a2cbffd881-b2d009efa8be824a.elb.us-east-1.amazonaws.com:6443 was refused - did you specify the right host or port? test/e2e/tests/test_helper.py:460: RuntimeError