self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7fed9e729970> def test_create_key_for_degraded_subscription(self): """API key creation succeeds for Degraded subscription.""" ns = _ns() subscription_name = "e2e-apikey-degraded-sub" auth_name = "e2e-apikey-degraded-auth" sa_name = "e2e-apikey-degraded-sa" missing_model = "nonexistent-model-apikey" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create with valid + missing model to trigger Degraded phase _create_test_subscription( subscription_name, [MODEL_REF, missing_model], users=[sa_user] ) _wait_reconcile(seconds=10) cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Degraded", f"Expected Degraded, got {phase}" # Create API key (should succeed) > api_key = _create_api_key( oc_token, name="degraded-sub-test", subscription=subscription_name ) test/e2e/tests/test_api_keys.py:1207: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InRfeVh6bHhqQS0wT2F5U1ZSbEpPWTgzNGJMM0NjRlBCWjBMc29kWi1nakEifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...57w3WjlGp59Mjd_VZTJAOraoOyeymEOLarlgDNuY87pJ4oc8iL1kVv_Gu-VIIqovXHYYevC2fneaN2wBuUqz8CZTkQLNxHKpMJn-Ky4FEGR_eFxpr3CN3g' name = 'degraded-sub-test', subscription = 'e2e-apikey-degraded-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7fed9e729dc0> def test_create_key_for_failed_subscription(self): """API key creation is rejected for Failed subscription to prevent key spam.""" ns = _ns() subscription_name = "e2e-apikey-failed-sub" auth_name = "e2e-apikey-failed-auth" sa_name = "e2e-apikey-failed-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Patch to Failed phase patch_data = { "status": { "phase": "Failed", "conditions": [{ "type": "Ready", "status": "False", "reason": "Failed", "message": "Test scenario", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], "modelRefStatuses": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Test failure" }] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Failed", f"Expected Failed, got {phase}" # Create API key (should be rejected for Failed subscriptions) resp = _create_api_key_raw( oc_token, name="failed-sub-test", subscription=subscription_name ) > assert resp.status_code == 403, \ f"Expected 403 Forbidden for Failed subscription, got {resp.status_code}: {resp.text}" E AssertionError: Expected 403 Forbidden for Failed subscription, got 500: E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_api_keys.py:1276: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7fed9e715520> def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. API key must be minted for this subscription oc_token = _get_cluster_token() > api_key = _create_api_key( oc_token, name=f"e2e-rate-limit-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) test/e2e/tests/test_subscription.py:466: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InRfeVh6bHhqQS0wT2F5U1ZSbEpPWTgzNGJMM0NjRlBCWjBMc29kWi1nakEifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...s9ojJ53ma23mmh1Nn1HLgwIhK8hdD8JEMlNa178Ow-kS3yb4Ytl6CpQ3oP2XrWOYGSOeiudhgtC8y-6puJ0wt4lw4fqXEq6JwMSxKYY71WYgpKMwnwoi9A' name = 'e2e-rate-limit-cbf47904' subscription = 'e2e-rate-limit-test-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7fed9e715a30> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() > api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) test/e2e/tests/test_subscription.py:587: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InRfeVh6bHhqQS0wT2F5U1ZSbEpPWTgzNGJMM0NjRlBCWjBMc29kWi1nakEifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...s9ojJ53ma23mmh1Nn1HLgwIhK8hdD8JEMlNa178Ow-kS3yb4Ytl6CpQ3oP2XrWOYGSOeiudhgtC8y-6puJ0wt4lw4fqXEq6JwMSxKYY71WYgpKMwnwoi9A' name = 'e2e-models-exempt-e930a86d' subscription = 'e2e-models-exempt-test-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestOrderingEdgeCases object at 0x7fed9e715070> def test_subscription_before_auth_policy(self): """Create subscription first, then auth policy -> should work once both exist.""" ns = _ns() try: # Subscription CR must exist before minting a key bound to it _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-ordering-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}]}], }, }) _wait_reconcile() _wait_for_maas_subscription_phase("e2e-ordering-sub", namespace=ns, timeout=90) > api_key = _create_api_key( _get_cluster_token(), name=f"e2e-ordering-{uuid.uuid4().hex[:8]}", subscription="e2e-ordering-sub", ) test/e2e/tests/test_subscription.py:985: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InRfeVh6bHhqQS0wT2F5U1ZSbEpPWTgzNGJMM0NjRlBCWjBMc29kWi1nakEifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...s9ojJ53ma23mmh1Nn1HLgwIhK8hdD8JEMlNa178Ow-kS3yb4Ytl6CpQ3oP2XrWOYGSOeiudhgtC8y-6puJ0wt4lw4fqXEq6JwMSxKYY71WYgpKMwnwoi9A' name = 'e2e-ordering-039f7a5f', subscription = 'e2e-ordering-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeError