self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7fbb34e548b0> def test_create_key_for_failed_subscription(self): """API key creation is rejected for Failed subscription to prevent key spam.""" ns = _ns() subscription_name = "e2e-apikey-failed-sub" auth_name = "e2e-apikey-failed-auth" sa_name = "e2e-apikey-failed-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Patch to Failed phase patch_data = { "status": { "phase": "Failed", "conditions": [{ "type": "Ready", "status": "False", "reason": "Failed", "message": "Test scenario", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], "modelRefStatuses": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Test failure" }] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Failed", f"Expected Failed, got {phase}" # Create API key (should be rejected for Failed subscriptions) resp = _create_api_key_raw( oc_token, name="failed-sub-test", subscription=subscription_name ) > assert resp.status_code == 403, \ f"Expected 403 Forbidden for Failed subscription, got {resp.status_code}: {resp.text}" E AssertionError: Expected 403 Forbidden for Failed subscription, got 500: E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_api_keys.py:1276: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7fbb349edc40> def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. API key must be minted for this subscription oc_token = _get_cluster_token() > api_key = _create_api_key( oc_token, name=f"e2e-rate-limit-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) test/e2e/tests/test_subscription.py:466: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImFwUEg5UW83aUxQSkJtNlU1R1ZtR0I0NVRaSXJkaGlfQk9kc1RPQ1I3aXMifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...XIdEBzUuvZE-Tql1LIYBRrvauRm-k5XtL2JYbM-S4n-wMJ8zwN6MQZtT6LgTSMrN8_XJquTw1sVGEyylSdYh_YvNpW5ICBGa0cFVNVDQns5Y6U3uKAzMbg' name = 'e2e-rate-limit-cf0c1150' subscription = 'e2e-rate-limit-test-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7fbb349ed850> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() > api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) test/e2e/tests/test_subscription.py:587: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImFwUEg5UW83aUxQSkJtNlU1R1ZtR0I0NVRaSXJkaGlfQk9kc1RPQ1I3aXMifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...XIdEBzUuvZE-Tql1LIYBRrvauRm-k5XtL2JYbM-S4n-wMJ8zwN6MQZtT6LgTSMrN8_XJquTw1sVGEyylSdYh_YvNpW5ICBGa0cFVNVDQns5Y6U3uKAzMbg' name = 'e2e-models-exempt-5dc8dd01' subscription = 'e2e-models-exempt-test-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestOrderingEdgeCases object at 0x7fbb349ed580> def test_subscription_before_auth_policy(self): """Create subscription first, then auth policy -> should work once both exist.""" ns = _ns() try: # Subscription CR must exist before minting a key bound to it _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-ordering-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}]}], }, }) _wait_reconcile() _wait_for_maas_subscription_phase("e2e-ordering-sub", namespace=ns, timeout=90) > api_key = _create_api_key( _get_cluster_token(), name=f"e2e-ordering-{uuid.uuid4().hex[:8]}", subscription="e2e-ordering-sub", ) test/e2e/tests/test_subscription.py:985: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImFwUEg5UW83aUxQSkJtNlU1R1ZtR0I0NVRaSXJkaGlfQk9kc1RPQ1I3aXMifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...XIdEBzUuvZE-Tql1LIYBRrvauRm-k5XtL2JYbM-S4n-wMJ8zwN6MQZtT6LgTSMrN8_XJquTw1sVGEyylSdYh_YvNpW5ICBGa0cFVNVDQns5Y6U3uKAzMbg' name = 'e2e-ordering-5375db6a', subscription = 'e2e-ordering-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7fbb34e9e730> def test_e2e_with_both_access_and_subscription_gets_200(self): """ Full E2E test: Create MaaSModelRef, MaaSAuthPolicy, and MaaSSubscription from scratch. API key with both access and subscription should get 200 OK. This is the comprehensive test that validates the complete E2E flow including MaaSModelRef creation and reconciliation. Other tests use existing models for speed. """ ns = _ns() model_ref = "e2e-test-model-success" auth_policy_name = "e2e-test-auth-success" subscription_name = "e2e-test-subscription-success" sa_name = "e2e-sa-success" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create model and governance resources together so the model # can reach Ready (requires MaaSSubscription + MaaSAuthPolicy). _create_test_maas_model(model_ref) _create_test_auth_policy(auth_policy_name, model_ref, users=[sa_user]) _create_test_subscription(subscription_name, model_ref, users=[sa_user]) endpoint = _wait_for_maas_model_ready(model_ref, timeout=120) # Extract path from endpoint (e.g., https://maas.../llm/facebook-opt-125m-simulated -> /llm/facebook-opt-125m-simulated) model_path = urlparse(endpoint).path # API key bound to this subscription at mint (inference does not send x-maas-subscription) > api_key = _create_api_key( oc_token, name=f"{sa_name}-key", subscription=subscription_name ) test/e2e/tests/test_subscription.py:1320: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImFwUEg5UW83aUxQSkJtNlU1R1ZtR0I0NVRaSXJkaGlfQk9kc1RPQ1I3aXMifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm..._4HrOiG53Y80sA_aET1ghrkXjdu7OmhZR4w7GT0gX3hj4QCVtKaTZpLWImzG6cC_GY8EZdBH6JP-4CuEJ45CsmQ92_CstIe2gNdymq_OL_T2eTY8oOeQvA' name = 'e2e-sa-success-key', subscription = 'e2e-test-subscription-success' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeError