self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f3089f13310> @pytest.mark.serial def test_create_key_for_pending_subscription(self): """API key creation succeeds for Pending subscription.""" ns = _ns() subscription_name = "e2e-apikey-pending-sub" auth_name = "e2e-apikey-pending-auth" sa_name = "e2e-apikey-pending-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_for_maas_subscription_phase(subscription_name, namespace=ns) # The controller never naturally produces Pending for # subscriptions (deriveFinalPhase returns Active/Degraded/Failed), # so a status patch is still necessary. Scale the controller down # first to eliminate the race that caused flakiness. _scale_controller_down() patch_data = { "status": { "phase": "Pending", "conditions": [{ "type": "Ready", "status": "False", "reason": "Pending", "message": "Reconciliation in progress", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Pending", f"Expected Pending, got {phase}" # Create API key (should succeed) api_key = _create_api_key( oc_token, name="pending-sub-test", subscription=subscription_name ) assert api_key is not None and api_key.startswith("sk-"), \ f"Expected valid API key, got: {api_key[:20] if api_key else None}" log.info("✅ API key created successfully for Pending subscription") finally: _delete_cr("maassubscription", subscription_name, namespace=ns) _delete_cr("maasauthpolicy", auth_name, namespace=ns) _delete_sa(sa_name, namespace=MODEL_NAMESPACE) try: _scale_controller_up() except Exception: log.exception("Best-effort controller scale-up failed") > _wait_for_cr_absent("maassubscription", subscription_name, namespace=ns) test/e2e/tests/test_api_keys.py:1571: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-apikey-pending-sub' namespace = 'models-as-a-service', timeout = 30, poll_interval = 2 def _wait_for_cr_absent(kind, name, namespace=None, timeout=30, poll_interval=2): """Wait until a CR is deleted (no longer found by the API server).""" namespace = namespace or _ns() deadline = time.time() + timeout while time.time() < deadline: if _get_cr(kind, name, namespace) is None: return time.sleep(poll_interval) > raise TimeoutError( f"{kind}/{name} in {namespace} still exists after {timeout}s" ) E TimeoutError: maassubscription/e2e-apikey-pending-sub in models-as-a-service still exists after 30s test/e2e/tests/test_helper.py:1552: TimeoutErrorself = <test_negative_security.TestAuthPolicyRemoval object at 0x7f3089ede4c0> @pytest.mark.serial def test_authpolicy_deletion_revokes_access(self): """Create auth policy, delete it, verify legacy per-model AuthPolicy is absent. Uses the unconfigured model to avoid interfering with other tests. In gateway-only mode, the controller should not create per-model Kuadrant AuthPolicies. This verifies the legacy per-model AuthPolicy remains absent before and after MaaSAuthPolicy deletion. This tests the controller's cleanup logic. Gateway enforcement of AuthPolicy is already covered by other tests (e.g. test_wrong_group_gets_403). """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-policy-{suffix}" model_ref = UNCONFIGURED_MODEL_REF kuadrant_auth_name = f"maas-auth-{model_ref}" try: # Create auth policy granting access > _create_test_auth_policy( policy_name, model_ref, groups=["system:authenticated"], ) test/e2e/tests/test_negative_security.py:450: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-neg-policy-f3fc517...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f3089e9b430> @pytest.mark.serial def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated > _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) test/e2e/tests/test_subscription.py:467: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-rate-limit-test-au...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f3089e9b670> @pytest.mark.serial def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated > _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) test/e2e/tests/test_subscription.py:588: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-models-exempt-test...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_subscription.TestMultipleSubscriptionsPerModel object at 0x7f3089e9b970> @pytest.mark.serial def test_user_in_one_of_two_subscriptions_gets_200(self): """Add a 2nd subscription for a different group. API key only in the original group should still get 200 (not blocked by the 2nd sub's group check).""" ns = _ns() try: > _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-extra-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "nonexistent-group-xyz"}]}, "modelRefs": [{"name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 999, "window": "1m"}]}], }, }) test/e2e/tests/test_subscription.py:696: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-extra-sub", "nam...: [{"name": "facebook-opt-125m-simulated", "namespace": "llm", "tokenRateLimits": [{"limit": 999, "window": "1m"}]}]}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...ahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessError