self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f2c792f7410> @pytest.mark.serial def test_reject_key_for_unreconciled_subscription(self): """ API key creation is rejected for unreconciled subscription (empty phase). Note: Temporarily sets webhook failurePolicy to Ignore to allow creating resources while controller is down, then restores to Fail. """ ns = _ns() subscription_name = "e2e-apikey-unreconciled-sub" auth_name = "e2e-apikey-unreconciled-auth" sa_name = "e2e-apikey-unreconciled-sa" webhook_name = "maas-validating-webhook-configuration" try: # Create service account and get token oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Temporarily set webhook failurePolicy to Ignore # This allows creates to succeed when controller/webhook is unavailable # Find webhook indices dynamically by name to avoid brittleness result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True, check=True ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Ignore"}) subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True, check=True ) # Scale down controller to prevent reconciliation _scale_controller_down() # Create resources (webhook unavailable but Ignore policy allows creates) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Verify subscription is unreconciled (empty phase) cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase", "") assert phase == "", f"Expected empty phase, got: {phase}" log.info("✅ Subscription is unreconciled (empty phase)") # Try to create API key (should fail with 400) response = requests.post( f"{_maas_api_url()}/v1/api-keys", headers={ "Authorization": f"Bearer {oc_token}", "Content-Type": "application/json" }, json={ "name": "unreconciled-sub-test", "subscription": subscription_name }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert response.status_code == 400, \ f"Expected 400 for unreconciled subscription, got {response.status_code}: {response.text}" response_data = response.json() assert "code" in response_data and response_data["code"] == "subscription_not_ready", \ f"Expected subscription_not_ready error code, got: {response_data}" log.info("✅ API key creation rejected for unreconciled subscription") finally: # Restore webhook failurePolicy to Fail # Wrap in try/except to guarantee controller scale-up and cleanup happen webhook_restore_error = None try: result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True ) if result.returncode != 0: raise RuntimeError( f"Failed to get webhook configuration {webhook_name} for restoration: {result.stderr}" ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Fail"}) if not patch_ops: raise RuntimeError( f"No matching webhooks found to restore in {webhook_name}. " f"Expected vmaassubscription.kb.io and vmaasauthpolicy.kb.io. " f"Found: {[w.get('name') for w in webhook_config.get('webhooks', [])]}" ) restore_result = subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True ) if restore_result.returncode != 0: raise RuntimeError( f"Webhook left in Ignore state! Failed to restore webhook {webhook_name} " f"with patch_ops {patch_ops}: {restore_result.stderr}" ) except Exception as e: webhook_restore_error = e log.error(f"Exception during webhook restoration: {e}") # Always scale controller back up and cleanup, even if webhook restore failed > _scale_controller_up() test/maas-e2e/test/e2e/tests/test_api_keys.py:1798: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:2121: in _scale_controller_up _scale_controller(1, namespace, timeout) test/maas-e2e/test/e2e/tests/test_helper.py:2073: in _scale_controller subprocess.run( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = None, capture_output = True, timeout = 60, check = True popenargs = (['oc', 'scale', 'deployment', 'maas-controller', '--replicas=1', '-n', ...],) kwargs = {'text': True, 'stdout': -1, 'stderr': -1} def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them, or pass capture_output=True to capture both. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'scale', 'deployment', 'maas-controller', '--replicas=1', '-n', 'opendatahub']' returned non-zero exit status 1. /usr/lib64/python3.11/subprocess.py:571: CalledProcessErrorself = <test_negative_security.TestAuthPolicyRemoval object at 0x7f2c792b5ed0> @pytest.mark.serial def test_authpolicy_deletion_revokes_access(self): """Create auth policy, delete it, verify legacy per-model AuthPolicy is absent. Uses the unconfigured model to avoid interfering with other tests. In gateway-only mode, the controller should not create per-model Kuadrant AuthPolicies. This verifies the legacy per-model AuthPolicy remains absent before and after MaaSAuthPolicy deletion. This tests the controller's cleanup logic. Gateway enforcement of AuthPolicy is already covered by other tests (e.g. test_wrong_group_gets_403). """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-policy-{suffix}" model_ref = UNCONFIGURED_MODEL_REF kuadrant_auth_name = f"maas-auth-{model_ref}" try: # Create auth policy granting access > _create_test_auth_policy( policy_name, model_ref, groups=["system:authenticated"], ) test/maas-e2e/test/e2e/tests/test_negative_security.py:467: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1102: in _create_test_auth_policy _apply_cr({ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'name': 'e2e-neg-policy-056a59bf...ok-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'users': [], 'groups': [{'name': 'system:authenticated'}]}}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for MaaSAuthPolicy models-as-a-service/e2e-neg-policy-056a59bf: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f2c79179550> @pytest.mark.serial def test_auth_pass_no_subscription_gets_403(self): """API key with auth pass but no matching subscription should get 403. The AuthPolicy includes a subscription-error-check rule that calls /internal/v1/subscriptions/select. If no subscription matches the user's groups, the request is denied with 403 "no matching subscription found for user". To test this, we temporarily add system:authenticated to the premium model's AuthPolicy (so auth passes) but keep the subscription only for premium-user (so subscription check fails). """ ns = _ns() api_key = _get_default_api_key() # First verify that default key currently gets 403 on premium model (auth fails) r = _inference(api_key, path=PREMIUM_MODEL_PATH) assert r.status_code == 403, f"Expected 403 for premium model (auth should fail), got {r.status_code}" # Now temporarily add system:authenticated to premium model's AuthPolicy try: # Get current auth policy and add system:authenticated group > _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-auth-pass-sub-fail", "namespace": ns}, "spec": { "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE}], "subjects": { "groups": [{"name": "system:authenticated"}], # Auth will pass }, }, }) test/maas-e2e/test/e2e/tests/test_subscription.py:582: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'name': 'e2e-auth-pass-sub-fail'...remium-simulated-simulated-premium', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for MaaSAuthPolicy models-as-a-service/e2e-auth-pass-sub-fail: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f2c79179dd0> @pytest.mark.serial def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated > _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) test/maas-e2e/test/e2e/tests/test_subscription.py:635: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1102: in _create_test_auth_policy _apply_cr({ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'name': 'e2e-rate-limit-test-aut...ok-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'users': [], 'groups': [{'name': 'system:authenticated'}]}}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for MaaSAuthPolicy models-as-a-service/e2e-rate-limit-test-auth: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f2c7917a690> @pytest.mark.serial def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" try: # 1. Create auth policy allowing system:authenticated > _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) test/maas-e2e/test/e2e/tests/test_subscription.py:755: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1102: in _create_test_auth_policy _apply_cr({ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'name': 'e2e-models-exempt-test-...ok-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'users': [], 'groups': [{'name': 'system:authenticated'}]}}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for MaaSAuthPolicy models-as-a-service/e2e-models-exempt-test-auth: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeError