/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7fe1814a5970> def test_central_models_endpoint_exempt_from_rate_limiting(self): """ Test that the central /v1/models endpoint remains accessible when token quota is exhausted. This test validates the end-to-end flow: 1. User exhausts token quota with inference requests (gets 429) 2. Central /v1/models endpoint is exempt at gateway level (gateway-default-deny TRLP) 3. Central endpoint calls model-specific /v1/models endpoints for discovery 4. Model-specific endpoints are also exempt (per-route TRLP fix) 5. Central endpoint successfully aggregates and returns model list This ensures the entire discovery chain works when quota is exhausted. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-central-models-exempt-auth" subscription_name = "e2e-central-models-exempt-sub" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 sa_name = f"e2e-central-models-exempt-sa-{uuid.uuid4().hex[:6]}" try: # 1. Create auth policy allowing system:authenticated log.info(f"Creating auth policy for {model_ref}") _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit log.info(f"Creating subscription with {token_limit} token limit") _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created and enforced _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription. # Use SA token to avoid environment-specific user-token 401s. oc_token = _create_sa_token(sa_name, namespace=_ns()) api_key = _create_api_key( oc_token, name=f"e2e-central-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) # 4. Exhaust the token limit # With 3 token limit and 5 requests, we're guaranteed to hit the limit # (each successful request consumes ≥1 token, so 5 requests > 3 token limit) max_requests = 5 success_count = 0 rate_limited = False log.info(f"Exhausting token quota: sending up to {max_requests} requests") for i in range(max_requests): r = _inference(api_key, path=model_path) request_num = i + 1 log.info(f"Request {request_num}: {r.status_code}") if r.status_code == 200: success_count += 1 elif r.status_code == 429: log.info(f"Rate limit hit after {success_count} successful requests") rate_limited = True break # Verify we hit rate limit (otherwise test setup is broken) assert rate_limited, \ f"Expected to hit rate limit within {max_requests} requests with {token_limit} token limit, " \ f"but got {success_count} successful requests without hitting limit" # 5. Verify inference is blocked log.info("Verifying inference endpoint is blocked...") r_inference = _inference(api_key, path=model_path) assert r_inference.status_code == 429, \ f"Expected 429 for inference after exhausting tokens, got {r_inference.status_code}" log.info("✓ Inference endpoint correctly blocked with 429") # 6. Verify central /v1/models endpoint still works log.info("Verifying central /v1/models endpoint is still accessible...") url = f"{_maas_api_url()}/v1/models" headers = {"Authorization": f"Bearer {api_key}"} r_models = requests.get(url, headers=headers, timeout=TIMEOUT, verify=TLS_VERIFY) > assert r_models.status_code == 200, \ f"Expected 200 for central /v1/models endpoint even when quota exhausted, got {r_models.status_code}. " \ f"The central /v1/models endpoint should be exempt from rate limiting (gateway-level) and " \ f"should be able to call model-specific /v1/models endpoints (per-route exemption). " \ f"Response: {r_models.text[:500]}" E AssertionError: Expected 200 for central /v1/models endpoint even when quota exhausted, got 500. The central /v1/models endpoint should be exempt from rate limiting (gateway-level) and should be able to call model-specific /v1/models endpoints (per-route exemption). Response: {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"001"} E assert 500 == 200 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_models_endpoint.py:2221: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_external_oidc.py:426: OIDC_USERNAME_NO_ACCESS and OIDC_PASSWORD_NO_ACCESS not configured