self = <test_subscription.TestSubscriptionEnforcement object at 0x7fee6a198cd0> def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. API key must be minted for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-rate-limit-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Send requests to exhaust the limit rate_limited = False success_count = 0 for i in range(total_requests): r = _inference(api_key, path=model_path, max_tokens=1) request_num = i + 1 log.info(f"Request {request_num}/{total_requests}: {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: rate_limited = True log.info(f"Rate limit exceeded after {success_count} successful requests") # Verify it's a rate limit 429, not a subscription error response_text = r.text.lower() if r.text else "" # Rate limit 429s typically mention "rate", "limit", or "quota" # Subscription 429s mention "subscription" without "rate" is_rate_limit_error = any(keyword in response_text for keyword in ["rate", "limit", "quota", "too many"]) is_subscription_error = "subscription" in response_text and not is_rate_limit_error assert is_rate_limit_error or not is_subscription_error, \ f"Expected rate limit 429, not subscription error. Response: {r.text[:500]}" # Check for Retry-After header (optional but good practice) retry_after = r.headers.get("Retry-After") or r.headers.get("retry-after") if retry_after: log.info(f"Retry-After header present: {retry_after}") break else: # Unexpected status code > raise AssertionError(f"Unexpected status {r.status_code} at request {request_num}: {r.text[:200]}") E AssertionError: Unexpected status 503 at request 1: no healthy upstream test/e2e/tests/test_subscription.py:506: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7fee6a198880> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Exhaust the token limit # With 3 token limit and 5 requests, we're guaranteed to hit the limit # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) max_requests = 5 success_count = 0 rate_limited = False log.info(f"Exhausting token quota: sending up to {max_requests} requests") for i in range(max_requests): r = _inference(api_key, path=model_path) request_num = i + 1 log.info(f"Request {request_num}: status {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: log.info(f"Rate limit hit after {success_count} successful requests") rate_limited = True break else: # Unexpected status during exhaustion log.warning(f"Unexpected status during quota exhaustion: {r.status_code}") # Verify we hit rate limit (otherwise test setup is broken) > assert rate_limited, \ f"Expected to hit rate limit within {max_requests} requests with {token_limit} token limit, " \ f"but got {success_count} successful requests without hitting limit" E AssertionError: Expected to hit rate limit within 5 requests with 3 token limit, but got 0 successful requests without hitting limit E assert False test/e2e/tests/test_subscription.py:617: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fee69cd5dc0> def test_central_models_endpoint_exempt_from_rate_limiting(self): """ Test that the central /v1/models endpoint remains accessible when token quota is exhausted. This test validates the end-to-end flow: 1. User exhausts token quota with inference requests (gets 429) 2. Central /v1/models endpoint is exempt at gateway level (gateway-default-deny TRLP) 3. Central endpoint calls model-specific /v1/models endpoints for discovery 4. Model-specific endpoints are also exempt (per-route TRLP fix) 5. Central endpoint successfully aggregates and returns model list This ensures the entire discovery chain works when quota is exhausted. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-central-models-exempt-auth" subscription_name = "e2e-central-models-exempt-sub" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated log.info(f"Creating auth policy for {model_ref}") _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit log.info(f"Creating subscription with {token_limit} token limit") _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created and enforced _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() api_key = _create_api_key( oc_token, name=f"e2e-central-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Exhaust the token limit # With 3 token limit and 5 requests, we're guaranteed to hit the limit # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) max_requests = 5 success_count = 0 rate_limited = False log.info(f"Exhausting token quota: sending up to {max_requests} requests") for i in range(max_requests): r = _inference(api_key, path=model_path) request_num = i + 1 log.info(f"Request {request_num}: {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: log.info(f"Rate limit hit after {success_count} successful requests") rate_limited = True break # Verify we hit rate limit (otherwise test setup is broken) > assert rate_limited, \ f"Expected to hit rate limit within {max_requests} requests with {token_limit} token limit, " \ f"but got {success_count} successful requests without hitting limit" E AssertionError: Expected to hit rate limit within 5 requests with 3 token limit, but got 0 successful requests without hitting limit E assert False test/e2e/tests/test_models_endpoint.py:2195: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_external_oidc.py:420: OIDC_USERNAME_NO_ACCESS and OIDC_PASSWORD_NO_ACCESS not configured