self = <test_subscription.TestStatusReporting object at 0x7fd8ee122a30> def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Scale down Kuadrant controller 2. Create subscription with valid model - TRLP created but not accepted 3. Wait for subscription to enter Degraded phase (TRLP ready=false) 4. Create API key and verify inference is blocked (403 Forbidden) 5. Scale Kuadrant controller back up 6. Wait for subscription to reach Active phase (TRLP ready=true) 7. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Scale down Kuadrant controller BEFORE creating subscription log.info("Step 1: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() time.sleep(5) # Give time for controller to fully stop # Step 2: Create auth policy and subscription log.info("Step 2: Creating subscription with Kuadrant controller down...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Wait for auth policy - will be Degraded since Kuadrant is down log.info("Waiting for MaaSAuthPolicy (will be Degraded with Kuadrant down)...") _wait_for_maas_auth_policy_phase(auth_name, "Degraded", timeout=60, require_auth_policies=True, require_enforced=False) # Step 3: Wait for subscription to reach Degraded phase with TRLP not ready log.info("Step 3: Waiting for subscription to enter Degraded phase (TRLP not ready)...") cr = _wait_for_maas_subscription_phase(subscription_name, "Degraded", timeout=120) _wait_for_subscription_trlp_status(subscription_name, expected_ready=False, timeout=120) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Degraded: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify at least one TRLP is not ready assert len(trlp_statuses) > 0, "Expected at least one TRLP status" assert any(not trlp.get("ready") for trlp in trlp_statuses), "Expected at least one TRLP to be not ready" log.info("✅ Subscription in Degraded phase with TRLP not ready") # Step 4: Create API key and verify inference is blocked log.info("Step 4: Creating API key and verifying inference is blocked...") api_key = _create_api_key(sa_token, name="e2e-trlp-test-key", subscription=subscription_name) resp = _inference(api_key, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID) assert resp.status_code == 403, f"Expected 403 Forbidden for Degraded subscription with TRLP not ready, got {resp.status_code}: {resp.text}" log.info("✅ Inference blocked for Degraded subscription with TRLP not ready") # Step 5: Scale Kuadrant controller back up log.info("Step 5: Scaling Kuadrant controller back up...") _scale_kuadrant_controller_up() time.sleep(10) # Give time for TRLP to reconcile and be accepted # Step 6: Wait for subscription to reach Active phase with TRLP ready log.info("Step 6: Waiting for subscription to reach Active phase (TRLP ready)...") _wait_for_maas_subscription_phase(subscription_name, "Active", timeout=120) _wait_for_subscription_trlp_status(subscription_name, expected_ready=True, timeout=120) cr = _get_cr("maassubscription", subscription_name, namespace=ns) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Active: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify all TRLPs are now ready assert all(trlp.get("ready") for trlp in trlp_statuses), "Expected all TRLPs to be ready" log.info("✅ Subscription returned to Active phase with all TRLPs ready") # Step 7: Verify inference works log.info("Step 7: Verifying inference works with Active subscription...") resp = _inference(api_key, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID) > assert resp.status_code == 200, f"Expected 200 OK for Active subscription, got {resp.status_code}: {resp.text}" E AssertionError: Expected 200 OK for Active subscription, got 503: no healthy upstream E assert 503 == 200 E + where 503 = <Response [503]>.status_code test/e2e/tests/test_subscription.py:2040: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_external_oidc.py:420: OIDC_USERNAME_NO_ACCESS and OIDC_PASSWORD_NO_ACCESS not configured