self = <test_tenant_health.TestTenantManagedResources object at 0x7f9cdef82fd0> kind = 'authpolicy.kuadrant.io', min_count = 1 @pytest.mark.parametrize("kind,min_count", [ ("httproute.gateway.networking.k8s.io", 1), ("authpolicy.kuadrant.io", 1), ]) def test_networking_resources_exist(self, kind, min_count): """Gateway API and policy resources must exist.""" items = _list_resources_by_label(kind, namespace=DEPLOYMENT_NAMESPACE) assert items is not None, f"Resource type '{kind}' not available on cluster" names = [item["metadata"]["name"] for item in items] > assert len(items) >= min_count, ( f"Expected >= {min_count} {kind}(s) with label {TENANT_LABEL_SELECTOR} " f"in namespace {DEPLOYMENT_NAMESPACE}, found {len(items)}: {names}" ) E AssertionError: Expected >= 1 authpolicy.kuadrant.io(s) with label maas.opendatahub.io/tenant-name=default-tenant in namespace opendatahub, found 0: [] E assert 0 >= 1 E + where 0 = len([]) test/e2e/tests/test_tenant_health.py:210: AssertionError/workspace/source/test/e2e/tests/test_tenant_health.py:248: CRD persesdashboards.perses.dev not registered on cluster/workspace/source/test/e2e/tests/test_tenant_health.py:248: CRD persesdatasources.perses.dev not registered on clusterself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f9cdef84be0> def test_create_key_for_failed_subscription(self): """API key creation is rejected for Failed subscription to prevent key spam.""" ns = _ns() subscription_name = "e2e-apikey-failed-sub" auth_name = "e2e-apikey-failed-auth" sa_name = "e2e-apikey-failed-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Patch to Failed phase patch_data = { "status": { "phase": "Failed", "conditions": [{ "type": "Ready", "status": "False", "reason": "Failed", "message": "Test scenario", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], "modelRefStatuses": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Test failure" }] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Failed", f"Expected Failed, got {phase}" # Create API key (should be rejected for Failed subscriptions) resp = _create_api_key_raw( oc_token, name="failed-sub-test", subscription=subscription_name ) > assert resp.status_code == 403, \ f"Expected 403 Forbidden for Failed subscription, got {resp.status_code}: {resp.text}" E AssertionError: Expected 403 Forbidden for Failed subscription, got 500: E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_api_keys.py:1276: AssertionErrorself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f9cdee0db80> def test_create_key_for_pending_subscription(self): """API key creation succeeds for Pending subscription.""" ns = _ns() subscription_name = "e2e-apikey-pending-sub" auth_name = "e2e-apikey-pending-auth" sa_name = "e2e-apikey-pending-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Patch to Pending phase patch_data = { "status": { "phase": "Pending", "conditions": [{ "type": "Ready", "status": "False", "reason": "Pending", "message": "Reconciliation in progress", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Pending", f"Expected Pending, got {phase}" # Create API key (should succeed) > api_key = _create_api_key( oc_token, name="pending-sub-test", subscription=subscription_name ) test/e2e/tests/test_api_keys.py:1328: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImlMQ3kxdkRjaS0xakNVZ0VsRjNJR2pna21xUFJfRUVvemtiS2llcnBaeTgifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...QqmbD0PCMfGez2MvKHFx9BT0exd87ONPuUZOBLGpSM1zAhVFmpGoVKRm53OVXBv758uhiLlRmyconezzIHeuJsyF48sqY5PbhurRZEJ6vnxfjC9kRIXUdQ' name = 'pending-sub-test', subscription = 'e2e-apikey-pending-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f9cdebbfaf0> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() _wait_for_maas_subscription_phase(subscription_name, timeout=90) # Wait for TRLP to be created AND enforced by Kuadrant/Limitador _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) # 3. Create API key for this subscription oc_token = _get_cluster_token() > api_key = _create_api_key( oc_token, name=f"e2e-models-exempt-{uuid.uuid4().hex[:8]}", subscription=subscription_name, ) test/e2e/tests/test_subscription.py:587: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImlMQ3kxdkRjaS0xakNVZ0VsRjNJR2pna21xUFJfRUVvemtiS2llcnBaeTgifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...IhPMaxwXKfRrllWAC-8YKLSxf2iQYRdfcG9CmjLuwYiwnGiPytQKYcliloQA8B47hMsG0CPB8QHJMdpNkFCutnv3Ts0yKTFNQMTpcOCmhXKkKiWJKxCrtQ' name = 'e2e-models-exempt-5a0d3116' subscription = 'e2e-models-exempt-test-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeErrorself = <test_subscription.TestOrderingEdgeCases object at 0x7f9cdef07340> def test_subscription_before_auth_policy(self): """Create subscription first, then auth policy -> should work once both exist.""" ns = _ns() try: # Subscription CR must exist before minting a key bound to it _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-ordering-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}]}], }, }) _wait_reconcile() _wait_for_maas_subscription_phase("e2e-ordering-sub", namespace=ns, timeout=90) > api_key = _create_api_key( _get_cluster_token(), name=f"e2e-ordering-{uuid.uuid4().hex[:8]}", subscription="e2e-ordering-sub", ) test/e2e/tests/test_subscription.py:985: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6ImlMQ3kxdkRjaS0xakNVZ0VsRjNJR2pna21xUFJfRUVvemtiS2llcnBaeTgifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...IhPMaxwXKfRrllWAC-8YKLSxf2iQYRdfcG9CmjLuwYiwnGiPytQKYcliloQA8B47hMsG0CPB8QHJMdpNkFCutnv3Ts0yKTFNQMTpcOCmhXKkKiWJKxCrtQ' name = 'e2e-ordering-57c3f214', subscription = 'e2e-ordering-sub' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 test/e2e/tests/test_helper.py:243: RuntimeError