self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f8774fdf610> @pytest.mark.serial def test_create_key_for_pending_subscription(self): """API key creation succeeds for Pending subscription.""" ns = _ns() subscription_name = "e2e-apikey-pending-sub" auth_name = "e2e-apikey-pending-auth" sa_name = "e2e-apikey-pending-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) > _wait_for_maas_subscription_phase(subscription_name, namespace=ns) test/e2e/tests/test_api_keys.py:1665: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-apikey-pending-sub', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 90, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=MAAS_SUBSCRIPTION_PHASE_TIMEOUT, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-apikey-pending-sub did not reach phase 'Active' within 90s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:1334: TimeoutErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f8774eee6d0> @pytest.mark.serial def test_rate_limit_exhaustion_gets_429(self): """ Test that a user gets 429 when they actually exceed their token rate limit. This test creates a dedicated subscription with a very low token limit, sends enough requests to exhaust it, and verifies a 429 response. Uses the unconfigured model to avoid interfering with other tests. """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-rate-limit-test-auth" subscription_name = "e2e-rate-limit-test-subscription" # Low limit so we exhaust it quickly. Actual tokens consumed per # response are non-deterministic (max_tokens is a ceiling, not exact), # so we send enough requests to be confident we hit the limit without # asserting exactly when the 429 arrives. token_limit = 10 window = "1m" total_requests = 15 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) > _wait_for_maas_subscription_phase(subscription_name) test/e2e/tests/test_subscription.py:594: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-rate-limit-test-subscription', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 90, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=MAAS_SUBSCRIPTION_PHASE_TIMEOUT, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-rate-limit-test-subscription did not reach phase 'Active' within 90s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:1334: TimeoutErrorself = <test_subscription.TestSubscriptionEnforcement object at 0x7f8774eee910> @pytest.mark.serial def test_models_endpoint_exempt_from_rate_limiting(self): """ Test that /v1/models endpoint remains accessible when token quota is exhausted. This verifies that users can discover model capabilities even when they've used all their inference tokens. The /v1/models endpoint is a discovery/metadata endpoint that does not consume tokens and should remain accessible. Ref: https://issues.redhat.com/browse/RHOAIENG-46770 Test steps: 1. Create subscription with very low token limit (15 tokens) 2. Exhaust the limit with inference requests (5 requests × 3 tokens = 15) 3. Verify inference requests get 429 (rate limited) 4. Verify /v1/models endpoint still returns 200 (not rate limited) """ # Use unconfigured model to isolate this test model_ref = UNCONFIGURED_MODEL_REF model_path = UNCONFIGURED_MODEL_PATH # Create unique subscription and auth policy names auth_policy_name = "e2e-models-exempt-test-auth" subscription_name = "e2e-models-exempt-test-subscription" # Very low limit for fast, deterministic test # With 3 token limit and max_tokens=1, we're guaranteed to exhaust quota within 5 requests # (even if each request uses exactly 1 token: 5 requests > 3 token limit) token_limit = 3 window = "1m" max_tokens = 1 try: # 1. Create auth policy allowing system:authenticated _create_test_auth_policy( name=auth_policy_name, model_refs=[model_ref], groups=["system:authenticated"] ) _wait_for_maas_auth_policy_phase(auth_policy_name, timeout=90, require_auth_policies=False) # 2. Create subscription with low token limit _create_test_subscription( name=subscription_name, model_refs=[model_ref], groups=["system:authenticated"], token_limit=token_limit, window=window ) > _wait_for_maas_subscription_phase(subscription_name, timeout=90) test/e2e/tests/test_subscription.py:715: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-models-exempt-test-subscription', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 90, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=MAAS_SUBSCRIPTION_PHASE_TIMEOUT, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-models-exempt-test-subscription did not reach phase 'Active' within 90s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:1334: TimeoutErrorself = <test_subscription.TestCascadeDeletion object at 0x7f8774ef4400> @pytest.mark.serial def test_delete_subscription_rebuilds_trlp(self): """Add a 2nd subscription, delete it -> TRLP rebuilt with only the original.""" ns = _ns() try: _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-temp-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{"name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 50, "window": "1m"}]}], }, }) > _wait_for_maas_subscription_phase("e2e-temp-sub") test/e2e/tests/test_subscription.py:916: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-temp-sub', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 90, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=MAAS_SUBSCRIPTION_PHASE_TIMEOUT, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-temp-sub did not reach phase 'Active' within 90s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:1334: TimeoutErrorself = <test_subscription.TestCascadeDeletion object at 0x7f8774ef4640> @pytest.mark.serial def test_trlp_persists_during_multi_subscription_deletion(self): """Validate CWE-693/CWE-400 fix: TRLP rebuilt in-place during deletion. Tests the fix for the security vulnerability where deleting one subscription would delete the entire TokenRateLimitPolicy, disabling rate limiting for ALL subscriptions to that model and creating a window for unthrottled requests. The fix ensures: 1. TRLP is rebuilt in-place when a subscription is deleted (not deleted entirely) 2. TRLP contains only remaining subscriptions after deletion 3. TRLP is deleted only when no subscriptions remain This prevents the rate-limit protection gap (CWE-693: Protection Mechanism Failure, CWE-400: Uncontrolled Resource Consumption). """ ns = _ns() trlp_ns = MODEL_NAMESPACE trlp_name = TRLP_NAME # Snapshot original subscription for restoration original_sub = _snapshot_cr("maassubscription", SIMULATOR_SUBSCRIPTION, ns) assert original_sub, f"Pre-existing {SIMULATOR_SUBSCRIPTION} not found" try: # Step 1: Create a second subscription for the same model log.info("Creating second subscription for the same model...") _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "e2e-second-sub", "namespace": ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 75, "window": "1m"}] }], }, }) > _wait_for_maas_subscription_phase("e2e-second-sub") test/e2e/tests/test_subscription.py:965: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-second-sub', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 90, require_model_statuses = False def _wait_for_maas_subscription_phase(name, expected_phase="Active", namespace=None, timeout=MAAS_SUBSCRIPTION_PHASE_TIMEOUT, require_model_statuses=False): """Wait for MaaSSubscription to reach a specific phase. Args: name: Name of the MaaSSubscription expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_model_statuses: If True, also requires modelRefStatuses to be populated (default: False). Set to True for status reporting tests. Returns: The subscription CR dict when the expected phase is reached Raises: TimeoutError: If MaaSSubscription doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) if phase == expected_phase: if require_model_statuses: expected_count = len(cr.get("spec", {}).get("modelRefs", [])) if len(model_statuses) >= expected_count: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}' with {len(model_statuses)}/{expected_count} modelRefStatuses") return cr else: log.info(f"MaaSSubscription {name} reached phase '{expected_phase}'") return cr log.debug(f"MaaSSubscription {name}: phase={phase}, modelRefStatuses={len(model_statuses)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSSubscription {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, modelRefStatuses={len(status.get('modelRefStatuses', []))})" ) E TimeoutError: MaaSSubscription e2e-second-sub did not reach phase 'Active' within 90s (current: phase=None, modelRefStatuses=0) test/e2e/tests/test_helper.py:1334: TimeoutError