/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7ff3afbcda90> def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Scale down Kuadrant controller 2. Create subscription with valid model - TRLP created but not accepted 3. Wait for subscription to enter Degraded phase (TRLP ready=false) 4. Create API key and verify inference is blocked (403 Forbidden) 5. Scale Kuadrant controller back up 6. Wait for subscription to reach Active phase (TRLP ready=true) 7. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Scale down Kuadrant controller BEFORE creating subscription log.info("Step 1: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() time.sleep(5) # Give time for controller to fully stop # Step 2: Create auth policy and subscription log.info("Step 2: Creating subscription with Kuadrant controller down...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Wait for auth policy to reconcile. In gateway-only mode, it remains Active even when # Kuadrant TRLP reconciliation is degraded. log.info("Waiting for MaaSAuthPolicy to reconcile...") _wait_for_maas_auth_policy_phase(auth_name, "Active", timeout=60, require_auth_policies=False) # Step 3: Wait for subscription to reach Degraded phase with TRLP not ready log.info("Step 3: Waiting for subscription to enter Degraded phase (TRLP not ready)...") cr = _wait_for_maas_subscription_phase(subscription_name, "Degraded", timeout=120) _wait_for_subscription_trlp_status(subscription_name, expected_ready=False, timeout=120) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Degraded: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify at least one TRLP is not ready assert len(trlp_statuses) > 0, "Expected at least one TRLP status" assert any(not trlp.get("ready") for trlp in trlp_statuses), "Expected at least one TRLP to be not ready" log.info("✅ Subscription in Degraded phase with TRLP not ready") # Step 4: Create API key and verify inference is blocked log.info("Step 4: Creating API key and verifying inference is blocked...") api_key = _create_api_key(sa_token, name="e2e-trlp-test-key", subscription=subscription_name) resp = _inference(api_key, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID) > assert resp.status_code == 403, f"Expected 403 Forbidden for Degraded subscription with TRLP not ready, got {resp.status_code}: {resp.text}" E AssertionError: Expected 403 Forbidden for Degraded subscription with TRLP not ready, got 503: no healthy upstream E assert 503 == 403 E + where 503 = <Response [503]>.status_code test/e2e/tests/test_subscription.py:1993: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_external_oidc.py:426: OIDC_USERNAME_NO_ACCESS and OIDC_PASSWORD_NO_ACCESS not configured