self = <test_negative_security.TestCrossModelAccess object at 0x7f184bb1e3a0> def test_key_cannot_access_model_outside_subscription(self): """Key for model A cannot infer on model B outside its subscription. Uses the pre-deployed unconfigured model (a model with no subscription granting access to it) to test cross-model access denial. """ api_key = _create_api_key(_get_cluster_token(), subscription=SIMULATOR_SUBSCRIPTION) # The unconfigured model exists but has no subscription granting access. # Using the same API key (bound to simulator-subscription which covers MODEL_REF) # should fail because the subscription doesn't cover UNCONFIGURED_MODEL_REF. r = _inference(api_key, path=UNCONFIGURED_MODEL_PATH) log.info("Cross-model access (model outside subscription) -> %s", r.status_code) > assert r.status_code in (401, 403), ( f"Expected 401 or 403 for model outside subscription scope, " f"got {r.status_code}: {r.text[:500]}" ) E AssertionError: Expected 401 or 403 for model outside subscription scope, got 503: no healthy upstream E assert 503 in (401, 403) E + where 503 = <Response [503]>.status_code test/e2e/tests/test_negative_security.py:230: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f184b85b370> def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. API key must be minted for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-rate-limit-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Send requests to exhaust the limit rate_limited = False success_count = 0 for i in range(total_requests): r = _inference(api_key, path=model_path, max_tokens=1) request_num = i + 1 log.info(f"Request {request_num}/{total_requests}: {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: rate_limited = True log.info(f"Rate limit exceeded after {success_count} successful requests") # Verify it's a rate limit 429, not a subscription error response_text = r.text.lower() if r.text else "" # Rate limit 429s typically mention "rate", "limit", or "quota" # Subscription 429s mention "subscription" without "rate" is_rate_limit_error = any(keyword in response_text for keyword in ["rate", "limit", "quota", "too many"]) is_subscription_error = "subscription" in response_text and not is_rate_limit_error assert is_rate_limit_error or not is_subscription_error, \ f"Expected rate limit 429, not subscription error. Response: {r.text[:500]}" # Check for Retry-After header (optional but good practice) retry_after = r.headers.get("Retry-After") or r.headers.get("retry-after") if retry_after: log.info(f"Retry-After header present: {retry_after}") break else: # Unexpected status code > raise AssertionError(f"Unexpected status {r.status_code} at request {request_num}: {r.text[:200]}") E AssertionError: Unexpected status 503 at request 1: no healthy upstream test/e2e/tests/test_subscription.py:505: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f184b85baf0> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90, require_auth_policies=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Exhaust the token limit # With 3 token limit and 5 requests, we're guaranteed to hit the limit # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) max_requests = 5 success_count = 0 rate_limited = False log.info(f"Exhausting token quota: sending up to {max_requests} requests") for i in range(max_requests): r = _inference(api_key, path=model_path) request_num = i + 1 log.info(f"Request {request_num}: status {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: log.info(f"Rate limit hit after {success_count} successful requests") rate_limited = True break else: # Unexpected status during exhaustion log.warning(f"Unexpected status during quota exhaustion: {r.status_code}") # Verify we hit rate limit (otherwise test setup is broken) > assert rate_limited, \ f"Expected to hit rate limit within {max_requests} requests with {token_limit} token limit, " \ f"but got {success_count} successful requests without hitting limit" E AssertionError: Expected to hit rate limit within 5 requests with 3 token limit, but got 0 successful requests without hitting limit E assert False test/e2e/tests/test_subscription.py:616: AssertionErrorself = <test_subscription.TestCascadeDeletion object at 0x7f184b847190> def test_unconfigured_model_denied_by_gateway_auth(self): """New model with no MaaSAuthPolicy/MaaSSubscription -> gateway default auth denies (403).""" # Precondition: unconfigured model fixture is deployed model = _get_cr("maasmodelref", UNCONFIGURED_MODEL_REF, namespace=MODEL_NAMESPACE) assert model is not None, ( f"MaaSModelRef {UNCONFIGURED_MODEL_REF} must exist in {MODEL_NAMESPACE} " f"(deploy test/e2e/fixtures/unconfigured first)" ) # Precondition: no per-route auth policy exists for this model assert not _cr_exists("maasauthpolicy", UNCONFIGURED_MODEL_REF, namespace=MODEL_NAMESPACE), ( f"MaaSAuthPolicy for {UNCONFIGURED_MODEL_REF} must NOT exist — " f"this test validates gateway-level deny-by-default" ) # Precondition: no subscription exists for this model assert not _cr_exists("maassubscription", UNCONFIGURED_MODEL_REF, namespace=MODEL_NAMESPACE), ( f"MaaSSubscription for {UNCONFIGURED_MODEL_REF} must NOT exist — " f"this test validates gateway-level deny-by-default" ) # Precondition: maas-gateway-auth is in place and accepted gw_auth = _get_cr("authpolicy", "maas-gateway-auth", namespace="openshift-ingress") assert gw_auth is not None, ( "maas-gateway-auth AuthPolicy must exist in openshift-ingress" ) conditions = gw_auth.get("status", {}).get("conditions", []) accepted = [c for c in conditions if c.get("type") == "Accepted"] assert accepted and accepted[0].get("status") == "True", ( f"maas-gateway-auth must be Accepted, got: {accepted}" ) # Verify deny-by-default: inference to unconfigured model should be denied api_key = _get_default_api_key() r = _inference(api_key, path=UNCONFIGURED_MODEL_PATH) log.info(f"Unconfigured model (no auth policy) -> {r.status_code}") > assert r.status_code == 403, f"Expected 403 (gateway default deny), got {r.status_code}" E AssertionError: Expected 403 (gateway default deny), got 503 E assert 503 == 403 E + where 503 = <Response [503]>.status_code test/e2e/tests/test_subscription.py:961: AssertionError/workspace/source/test/e2e/tests/test_subscription.py:1036: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7f184bcd7c40> def test_explicit_subscription_header(self): """ Test: K8s token with multiple subscriptions can list models by providing x-maas-subscription header. Expected: HTTP 200 with models from only the specified subscription. Note: Creates SA that has access to both simulator-subscription (via system:authenticated) and premium-simulator-subscription (by adding SA to its users list). Uses K8s token directly (not API key) since API keys ignore the header. """ sa_name = "e2e-models-explicit-header-sa" sa_ns = "default" maas_ns = _ns() sa_user = None try: # Create service account - will be in system:authenticated group # This gives access to simulator-subscription automatically sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Add SA to premium-simulator-subscription to give it access to a second subscription log.info(f"Adding {sa_user} to premium-simulator-subscription users") subprocess.run([ "oc", "patch", "maassubscription", PREMIUM_SIMULATOR_SUBSCRIPTION, "-n", maas_ns, "--type=merge", "-p", json.dumps({"spec": {"owner": {"users": [sa_user]}}}) ], check=True) _wait_reconcile() # Test: GET /v1/models WITH x-maas-subscription header using K8s token # Expected: Returns models from simulator-subscription only log.info("Testing: GET /v1/models with K8s token and explicit subscription header: simulator-subscription") url = f"{_maas_api_url()}/v1/models" r = _request_with_gateway_retry( requests.get, url, headers={ "Authorization": f"Bearer {sa_token}", # K8s token, not API key "x-maas-subscription": SIMULATOR_SUBSCRIPTION, }, ) assert r.status_code == 200, f"Expected 200 with explicit subscription header, got {r.status_code}: {r.text}" # Validate response structure data = r.json() assert data.get("object") == "list", f"Expected object='list', got {data.get('object')}" assert "data" in data, "Response missing 'data' field" models = data.get("data", []) if data.get("data") is not None else [] # Should have at least one model from simulator-subscription > assert len(models) > 0, f"Expected at least one model in response, got {len(models)}. Data was: {data.get('data')}" E AssertionError: Expected at least one model in response, got 0. Data was: [] E assert 0 > 0 E + where 0 = len([]) test/e2e/tests/test_models_endpoint.py:431: AssertionError