/workspace/source/test/e2e/tests/test_api_keys.py:930: CronJob maas-api-key-cleanup not found in opendatahub: Error from server (NotFound): cronjobs.batch "maas-api-key-cleanup" not found/workspace/source/test/e2e/tests/test_api_keys.py:977: NetworkPolicy maas-api-cleanup-restrict not found in opendatahub: Error from server (NotFound): networkpolicies.networking.k8s.io "maas-api-cleanup-restrict" not found/workspace/source/test/e2e/tests/test_api_keys.py:1096: Cannot find maas-api pod in opendatahub: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f4190e259a0> def test_create_key_for_degraded_subscription(self): """API key creation succeeds for Degraded subscription.""" ns = _ns() subscription_name = "e2e-apikey-degraded-sub" auth_name = "e2e-apikey-degraded-auth" sa_name = "e2e-apikey-degraded-sa" missing_model = "nonexistent-model-apikey" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create with valid + missing model to trigger Degraded phase _create_test_subscription( subscription_name, [MODEL_REF, missing_model], users=[sa_user] ) _wait_reconcile(seconds=10) cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") > assert phase == "Degraded", f"Expected Degraded, got {phase}" E AssertionError: Expected Degraded, got None E assert None == 'Degraded' test/e2e/tests/test_api_keys.py:1216: AssertionErrorself = <test_api_keys.TestAPIKeySubscriptionFilter object at 0x7f4190d87c10> api_keys_base_url = 'https://maas.apps.db4f30b5-fe94-4a15-a0aa-91d680cc2069.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjVFU0ZWb1RoLXFybUZPQkhlQnZaMWFXbm9QcnhtWV9WRVZYenlEVzVWaTQifQ.e...0m1AXsHh_bKSCXsgIzwqrglKxhhJXj2jbF4e23UF3MqI7JoZE2evDyhFYreEH19cFriug_z5RfF-MUhyQ', 'Content-Type': 'application/json'} def test_search_filters_by_subscription(self, api_keys_base_url: str, headers: dict): """Search with subscription filter returns only keys bound to that subscription.""" sub_a = f"e2e-filter-sub-a-{os.urandom(4).hex()}" sub_b = f"e2e-filter-sub-b-{os.urandom(4).hex()}" ns = _ns() sa_name = f"e2e-filter-sa-{os.urandom(4).hex()}" key_ids_a = [] key_ids_b = [] try: # Create one SA authorized for both subscriptions so that # exclusion in search results is attributable to the subscription # filter, not user-scoping. oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) sa_headers = {"Authorization": f"Bearer {oc_token}", "Content-Type": "application/json"} _create_test_auth_policy(f"{sub_a}-auth", MODEL_REF, users=[sa_user]) _create_test_subscription(sub_a, MODEL_REF, users=[sa_user]) _wait_for_maas_subscription_phase(sub_a, namespace=ns) _create_test_auth_policy(f"{sub_b}-auth", MODEL_REF, users=[sa_user]) _create_test_subscription(sub_b, MODEL_REF, users=[sa_user]) > _wait_for_maas_subscription_phase(sub_b, namespace=ns) test/e2e/tests/test_api_keys.py:1511: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-filter-sub-b-ec9141ba', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 60, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=60, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-filter-sub-b-ec9141ba did not reach phase 'Active' within 60s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:760: TimeoutError/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestMissingModelRef object at 0x7f4190ea5a90> def test_subscription_with_nonexistent_model_ref(self): """MaaSSubscription generates TRLP only for valid model, not ghost model. Creates a subscription referencing one valid model and one ghost model, waits for Degraded phase, then asserts that a TRLP exists for the valid model but not for the ghost model. """ suffix = uuid.uuid4().hex[:8] sub_name = f"e2e-neg-ghost-sub-{suffix}" auth_name = f"e2e-neg-ghost-sub-auth-{suffix}" ghost_model = f"nonexistent-model-{suffix}" try: _create_test_auth_policy(auth_name, MODEL_REF, groups=["system:authenticated"]) _create_test_subscription( sub_name, [MODEL_REF, ghost_model], groups=["system:authenticated"], ) > _wait_for_maas_subscription_phase(sub_name, "Degraded", timeout=60) test/e2e/tests/test_negative_security.py:336: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-neg-ghost-sub-37068491', expected_phase = 'Degraded' namespace = 'models-as-a-service', timeout = 60, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=60, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-neg-ghost-sub-37068491 did not reach phase 'Degraded' within 60s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:760: TimeoutErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f4190ee1520> def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_reconcile() # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) _wait_reconcile() # Wait for TRLP to be created AND enforced by Kuadrant/Limitador. # Without this, requests bypass token rate limiting entirely. > _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=90) test/e2e/tests/test_subscription.py:480: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'e2e-unconfigured-facebook-opt-125m-simulated' model_namespace = 'llm', timeout = 90 def _wait_for_token_rate_limit_policy(model_ref, model_namespace=MODEL_NAMESPACE, timeout=60): """Wait for TokenRateLimitPolicy to be created and enforced for a model. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) conditions = trlp.get("status", {}).get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) if enforced and enforced.get("status") == "True": log.info(f"TokenRateLimitPolicy {trlp_name} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} exists but not enforced yet") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-e2e-unconfigured-facebook-opt-125m-simulated was not created and enforced in llm within 90s test/e2e/tests/test_helper.py:712: TimeoutErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f4190ee17c0> def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90, require_auth_policies=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) > _wait_for_maas_subscription_phase(subscription_name, timeout=90) test/e2e/tests/test_subscription.py:596: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-models-exempt-test-subscription', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 90, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=60, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-models-exempt-test-subscription did not reach phase 'Active' within 90s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:760: TimeoutError