/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_subscription.TestSubscriptionEnforcement object at 0x7f3fd54e3790> def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. API key must be minted for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-rate-limit-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Send requests to exhaust the limit rate_limited = False success_count = 0 for i in range(total_requests): r = _inference(api_key, path=model_path, max_tokens=1) request_num = i + 1 log.info(f"Request {request_num}/{total_requests}: {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: rate_limited = True log.info(f"Rate limit exceeded after {success_count} successful requests") # Verify it's a rate limit 429, not a subscription error response_text = r.text.lower() if r.text else "" # Rate limit 429s typically mention "rate", "limit", or "quota" # Subscription 429s mention "subscription" without "rate" is_rate_limit_error = any(keyword in response_text for keyword in ["rate", "limit", "quota", "too many"]) is_subscription_error = "subscription" in response_text and not is_rate_limit_error assert is_rate_limit_error or not is_subscription_error, \ f"Expected rate limit 429, not subscription error. Response: {r.text[:500]}" # Check for Retry-After header (optional but good practice) retry_after = r.headers.get("Retry-After") or r.headers.get("retry-after") if retry_after: log.info(f"Retry-After header present: {retry_after}") break else: # Unexpected status code > raise AssertionError(f"Unexpected status {r.status_code} at request {request_num}: {r.text[:200]}") E AssertionError: Unexpected status 403 at request 1: subscription rate limiting policies are not ready test/e2e/tests/test_subscription.py:524: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f3fd54e3160> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90, require_auth_policies=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Exhaust the token limit # With 3 token limit and 5 requests, we're guaranteed to hit the limit # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) max_requests = 5 success_count = 0 rate_limited = False log.info(f"Exhausting token quota: sending up to {max_requests} requests") for i in range(max_requests): r = _inference(api_key, path=model_path) request_num = i + 1 log.info(f"Request {request_num}: status {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: log.info(f"Rate limit hit after {success_count} successful requests") rate_limited = True break else: # Unexpected status during exhaustion log.warning(f"Unexpected status during quota exhaustion: {r.status_code}") # Verify we hit rate limit (otherwise test setup is broken) > assert rate_limited, \ f"Expected to hit rate limit within {max_requests} requests with {token_limit} token limit, " \ f"but got {success_count} successful requests without hitting limit" E AssertionError: Expected to hit rate limit within 5 requests with 3 token limit, but got 5 successful requests without hitting limit E assert False test/e2e/tests/test_subscription.py:633: AssertionErrorself = <test_subscription.TestMultipleAuthPoliciesPerModel object at 0x7f3fd54e3280> def test_two_auth_policies_or_logic(self): """Two auth policies for the premium model with OR logic: user matching either gets access.""" ns = _ns() try: # Create a 2nd auth policy that allows system:authenticated (user's actual group) _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-premium-sa-auth", "namespace": ns}, "spec": { "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }) # Create a subscription for system:authenticated on premium model _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-premium-sa-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}]}], }, }) _wait_reconcile() # Key must be minted for the premium subscription > api_key = _create_api_key( _get_cluster_token(), name=f"e2e-premium-sa-{uuid.uuid4().hex[:8]}", subscription="e2e-premium-sa-sub", ) test/e2e/tests/test_subscription.py:736: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6Ii1HM2xfQkVab0c5RzluZ3NjWXlJRzlsU2lpaG12QkF5aFVKMjZGdnp3UUEifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...PW6vp7gYb_Je8OcAgH5tiFkPd5GGPO2vUuhVKI6eoOMr0a1yN3olDmZZQOM15NGzLNjuqXFtrEoBf5ugs6w6-TtcMBKp-Hw1eQBpuf_fdhM1Jfw-BhaRXw' name = 'e2e-premium-sa-3af9a85a', subscription = 'e2e-premium-sa-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 400 {"code":"subscription_not_ready","error":"subscription is unreconciled (no status.phase set)"} test/e2e/tests/test_helper.py:276: RuntimeErrorself = <test_subscription.TestCascadeDeletion object at 0x7f3fd5518730> def test_trlp_persists_during_multi_subscription_deletion(self): """Validate CWE-693/CWE-400 fix: TRLP rebuilt in-place during deletion. Tests the fix for the security vulnerability where deleting one subscription would delete the entire TokenRateLimitPolicy, disabling rate limiting for ALL subscriptions to that model and creating a window for unthrottled requests. The fix ensures: 1. TRLP is rebuilt in-place when a subscription is deleted (not deleted entirely) 2. TRLP contains only remaining subscriptions after deletion 3. TRLP is deleted only when no subscriptions remain This prevents the rate-limit protection gap (CWE-693: Protection Mechanism Failure, CWE-400: Uncontrolled Resource Consumption). """ ns = _ns() trlp_ns = MODEL_NAMESPACE trlp_name = TRLP_NAME # Snapshot original subscription for restoration original_sub = _snapshot_cr("maassubscription", SIMULATOR_SUBSCRIPTION, ns) assert original_sub, f"Pre-existing {SIMULATOR_SUBSCRIPTION} not found" try: # Step 1: Create a second subscription for the same model log.info("Creating second subscription for the same model...") _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-second-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 75, "window": "1m"}] }], }, }) _wait_reconcile() # Step 2: Verify TRLP exists and contains both subscriptions log.info("Verifying TRLP contains both subscriptions...") trlp_with_both = _get_cr("tokenratelimitpolicy", trlp_name, trlp_ns) assert trlp_with_both, f"TRLP {trlp_name} not found in {trlp_ns} after creating 2nd subscription" # Verify both subscriptions are in the TRLP limits limits = trlp_with_both.get("spec", {}).get("limits", {}) assert limits, f"TRLP {trlp_name} has no limits defined" # Look for both subscription references in TRLP limits # Format: {namespace}-{subscription-name}-{model-name}-tokens simulator_limit_key = f"{ns.replace('/', '-')}-{SIMULATOR_SUBSCRIPTION}-{MODEL_REF}-tokens" second_limit_key = f"{ns.replace('/', '-')}-e2e-second-sub-{MODEL_REF}-tokens" assert simulator_limit_key in limits, \ f"Original subscription limit key '{simulator_limit_key}' not found in TRLP. Available keys: {list(limits.keys())}" > assert second_limit_key in limits, \ f"Second subscription limit key '{second_limit_key}' not found in TRLP. Available keys: {list(limits.keys())}" E AssertionError: Second subscription limit key 'models-as-a-service-e2e-second-sub-facebook-opt-125m-simulated-tokens' not found in TRLP. Available keys: ['models-as-a-service-simulator-subscription-facebook-opt-125m-simulated-tokens'] E assert 'models-as-a-service-e2e-second-sub-facebook-opt-125m-simulated-tokens' in {'models-as-a-service-simulator-subscription-facebook-opt-125m-simulated-tokens': {'counters': [{'expression': 'auth.identity.userid'}], 'rates': [{'limit': 100, 'window': '1m'}], 'when': [{'predicate': 'auth.identity.selected_subscription_key == "models-as-a-service/simulator-subscription@llm/facebook-opt-125m-simulated" && !request.path.endsWith("/v1/models")'}]}} test/e2e/tests/test_subscription.py:859: AssertionError/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdcls = <class 'test_subscription.TestE2ESubscriptionFlow'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests. This validates that expected resources exist and are in the correct state. Tests will FAIL (not skip) if prerequisites are missing, ensuring CI catches issues. """ log.info("=" * 60) log.info("Validating E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'facebook-opt-125m-simulated' not Ready (phase=Pending, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_subscription.py:1282: Failed