self = <test_tenant_health.TestTenantManagedResources object at 0x7fedb1c6c070> def test_authpolicy_exists(self): """AuthPolicy must exist (may be in gateway namespace, not deployment namespace).""" items = _list_resources_by_label("authpolicy.kuadrant.io", namespace=DEPLOYMENT_NAMESPACE) if items: return items = _list_resources_by_label("authpolicy.kuadrant.io") assert items is not None, "AuthPolicy CRD (kuadrant.io) not available on cluster" names = [item["metadata"]["name"] for item in items] > assert len(items) >= 1, ( f"Expected >= 1 AuthPolicy with label {TENANT_LABEL_SELECTOR}, " f"found {len(items)}: {names}" ) E AssertionError: Expected >= 1 AuthPolicy with label maas.opendatahub.io/tenant-name=default-tenant, found 0: [] E assert 0 >= 1 E + where 0 = len([]) test/e2e/tests/test_tenant_health.py:219: AssertionError/workspace/source/test/e2e/tests/test_tenant_health.py:257: CRD persesdashboards.perses.dev not registered on cluster/workspace/source/test/e2e/tests/test_tenant_health.py:257: CRD persesdatasources.perses.dev not registered on clusterself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7fedb1ca18b0> def test_create_key_for_pending_subscription(self): """API key creation succeeds for Pending subscription.""" ns = _ns() subscription_name = "e2e-apikey-pending-sub" auth_name = "e2e-apikey-pending-auth" sa_name = "e2e-apikey-pending-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Patch to Pending phase patch_data = { "status": { "phase": "Pending", "conditions": [{ "type": "Ready", "status": "False", "reason": "Pending", "message": "Reconciliation in progress", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Pending", f"Expected Pending, got {phase}" # Create API key (should succeed) > api_key = _create_api_key( oc_token, name="pending-sub-test", subscription=subscription_name ) test/e2e/tests/test_api_keys.py:1328: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InBTc0xTSHJkYm9FZ0hVLXRDTFd5NHQ5Y3htWmU0QWY4d3VrSHJ2am1Vd3cifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...L7EoRhvZK6tJ_Lq8f7Qn1YIRURpgqB0Ecd7e6_pPiVDBLlGv2gHDvQODXDk-k2lSmnNuBm9Txxn_2u7omLgWVJza5pzSnLQc25HAc-_ekZa_6hMpnc35gw' name = 'pending-sub-test', subscription = 'e2e-apikey-pending-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7fedb1bbcf70> def test_reject_key_for_unreconciled_subscription(self): """ API key creation is rejected for unreconciled subscription (empty phase). Note: Temporarily sets webhook failurePolicy to Ignore to allow creating resources while controller is down, then restores to Fail. """ ns = _ns() subscription_name = "e2e-apikey-unreconciled-sub" auth_name = "e2e-apikey-unreconciled-auth" sa_name = "e2e-apikey-unreconciled-sa" webhook_name = "maas-validating-webhook-configuration" try: # Create service account and get token oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Temporarily set webhook failurePolicy to Ignore # This allows creates to succeed when controller/webhook is unavailable # Find webhook indices dynamically by name to avoid brittleness result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True, check=True ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Ignore"}) subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True, check=True ) # Scale down controller to prevent reconciliation _scale_controller_down() # Create resources (webhook unavailable but Ignore policy allows creates) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Verify subscription is unreconciled (empty phase) cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase", "") assert phase == "", f"Expected empty phase, got: {phase}" log.info("✅ Subscription is unreconciled (empty phase)") # Try to create API key (should fail with 400) response = requests.post( f"{_maas_api_url()}/v1/api-keys", headers={ "Authorization": f"Bearer {oc_token}", "Content-Type": "application/json" }, json={ "name": "unreconciled-sub-test", "subscription": subscription_name }, timeout=TIMEOUT, verify=TLS_VERIFY, ) > assert response.status_code == 400, \ f"Expected 400 for unreconciled subscription, got {response.status_code}: {response.text}" E AssertionError: Expected 400 for unreconciled subscription, got 500: E assert 500 == 400 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_api_keys.py:1408: AssertionErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7fedb1b7e310> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() > api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) test/e2e/tests/test_subscription.py:587: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InBTc0xTSHJkYm9FZ0hVLXRDTFd5NHQ5Y3htWmU0QWY4d3VrSHJ2am1Vd3cifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...j1egy4xF0W-0eLz01MNPfiFIFaikeV6qfAouE5Cn0gU4GQ94ikz280MaQaq1dd08Hea0OIT9zD1PokVsV3mo-M4Wal0bwAzRGoFgCbQA1aeVvSglWPw4uQ' name = 'e2e-models-exempt-096f4352' subscription = 'e2e-models-exempt-test-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestOrderingEdgeCases object at 0x7fedb1673190> def test_subscription_before_auth_policy(self): """Create subscription first, then auth policy -> should work once both exist.""" ns = _ns() try: # Subscription CR must exist before minting a key bound to it _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-ordering-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}]}], }, }) _wait_reconcile() _wait_for_maas_subscription_phase("e2e-ordering-sub", namespace=ns, timeout=90) > api_key = _create_api_key( _get_cluster_token(), name=f"e2e-ordering-{uuid.uuid4().hex[:8]}", subscription="e2e-ordering-sub", ) test/e2e/tests/test_subscription.py:985: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6InBTc0xTSHJkYm9FZ0hVLXRDTFd5NHQ5Y3htWmU0QWY4d3VrSHJ2am1Vd3cifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...j1egy4xF0W-0eLz01MNPfiFIFaikeV6qfAouE5Cn0gU4GQ94ikz280MaQaq1dd08Hea0OIT9zD1PokVsV3mo-M4Wal0bwAzRGoFgCbQA1aeVvSglWPw4uQ' name = 'e2e-ordering-1fa96989', subscription = 'e2e-ordering-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeError