/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:321: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:378: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:454: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7f3393141130> def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Scale down Kuadrant controller 2. Create subscription with valid model - TRLP created but not accepted 3. Wait for subscription to enter Degraded phase (TRLP ready=false) 4. Create API key and verify inference is blocked (403 Forbidden) 5. Scale Kuadrant controller back up 6. Wait for subscription to reach Active phase (TRLP ready=true) 7. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Scale down Kuadrant controller BEFORE creating subscription log.info("Step 1: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() time.sleep(5) # Give time for controller to fully stop # Step 2: Create auth policy and subscription log.info("Step 2: Creating subscription with Kuadrant controller down...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Wait for auth policy to reconcile. In gateway-only mode, it remains Active even when # Kuadrant TRLP reconciliation is degraded. log.info("Waiting for MaaSAuthPolicy to reconcile...") _wait_for_maas_auth_policy_phase(auth_name, "Active", timeout=60, require_auth_policies=False) # Step 3: Wait for subscription to reach Degraded phase with TRLP not ready log.info("Step 3: Waiting for subscription to enter Degraded phase (TRLP not ready)...") cr = _wait_for_maas_subscription_phase(subscription_name, "Degraded", timeout=120) _wait_for_subscription_trlp_status(subscription_name, expected_ready=False, timeout=120) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Degraded: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify at least one TRLP is not ready assert len(trlp_statuses) > 0, "Expected at least one TRLP status" assert any(not trlp.get("ready") for trlp in trlp_statuses), "Expected at least one TRLP to be not ready" log.info("✅ Subscription in Degraded phase with TRLP not ready") # Step 4: Create API key and verify inference is blocked log.info("Step 4: Creating API key and verifying inference is blocked...") api_key = _create_api_key(sa_token, name="e2e-trlp-test-key", subscription=subscription_name) resp = _inference(api_key, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID) > assert resp.status_code == 403, f"Expected 403 Forbidden for Degraded subscription with TRLP not ready, got {resp.status_code}: {resp.text}" E AssertionError: Expected 403 Forbidden for Degraded subscription with TRLP not ready, got 503: no healthy upstream E assert 503 == 403 E + where 503 = <Response [503]>.status_code test/e2e/tests/test_subscription.py:1993: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f33930dc760> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: apply_gateway_fixture(case["gateway_name"], fixture_label=case["tenant_label_name"]) bootstrap_aitenant_tenant(case) tenant = wait_for_json("tenant", TENANT_CR_NAME, case["tenant_ns"], timeout=180) assert tenant["spec"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") > wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase=("Active", "Degraded"), ) test/e2e/tests/test_multi_tenant_integration.py:77: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:253: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-sub-aecf84b0' namespace = 'e2e-mt-aecf84b0' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maassubscription/e2e-sub-aecf84b0 in e2e-mt-aecf84b0 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSSubscription', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSSubscription","metadata":{"annotations":{},"name":"e2e-sub-aecf84b0","namespace":"e2e-mt-aecf84b0"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm","tokenRateLimits":[{"limit":100,"window":"1m"}]}],"owner":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-06-11T23:49:00Z', 'finalizers': ['maas.opendatahub.io/subscription-cleanup'], 'generation': 1, 'name': 'e2e-sub-aecf84b0', 'namespace': 'e2e-mt-aecf84b0', 'resourceVersion': '52139', 'uid': '98443d1e-bb95-4cbb-837a-fb11f3a74c04'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm', 'tokenRateLimits': [{'limit': 100, 'window': '1m'}]}], 'owner': {'groups': [{'name': 'system:authenticated'}]}, 'priority': 0}, 'status': {'conditions': [{'lastTransitionTime': '2026-06-11T23:49:00Z', 'message': '', 'observedGeneration': 1, 'reason': 'NoDuplicatePeers', 'status': 'False', 'type': 'SpecPriorityDuplicate'}, {'lastTransitionTime': '2026-06-11T23:49:00Z', 'message': 'failed to reconcile TokenRateLimitPolicies: model llm/facebook-opt-125m-simulated is not attached to tenant gateway for subscription e2e-mt-aecf84b0/e2e-sub-aecf84b0: HTTPRoute llm/facebook-opt-125m-simulated-kserve-route does not reference tenant Gateway openshift-ingress/e2e-mt-aecf84b0', 'observedGeneration': 1, 'reason': 'ReconcileFailed', 'status': 'False', 'type': 'Ready'}], 'modelRefStatuses': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm', 'ready': True, 'reason': 'Valid'}], 'phase': 'Failed'}} test/e2e/tests/multitenancy_helpers.py:198: AssertionError/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:68: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:87: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:96: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:113: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:124: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:109: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:122: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:130: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:137: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:153: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:176: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_subscription_isolation.py:101: S4 tenant subscription isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_subscription_isolation.py:120: S4 tenant subscription isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_rate_limit_isolation.py:154: S4 tenant rate-limit isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_rate_limit_isolation.py:163: S4 tenant rate-limit isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation landsself = <test_external_oidc.TestOIDCTokenFlow object at 0x7f339332a1c0> maas_api_base_url = 'https://maas.apps.2be519f3-a56a-4801-9821-e5615bf56ec5.prod.konfluxeaas.com/maas-api' def test_oidc_token_can_create_api_key(self, maas_api_base_url: str): """OIDC token from default user (alice_lead) can mint an API key.""" token = _request_oidc_token() > data = _create_oidc_api_key(maas_api_base_url, token) test/e2e/tests/test_external_oidc.py:207: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.2be519f3-a56a-4801-9821-e5615bf56ec5.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJxVDJDY1JhYl9pdDEzc3BOdGRCeDNHUUd2MFRYMjNiUzA0aG5xcDFyVnhBIn0.eyJle...oSs9UCKXQ9MLHyVxk3-rOfqC4dm7R7QHdA-ZpVk4EwaGoSB4FNXKwVkZNnoua6jwprsbka41qy0-CZ1QQV4VYbMWIDGrYu7m-M38Fxc7mVDQGEOP1X5ALQ' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCMultiUser object at 0x7f33935d9940> maas_api_base_url = 'https://maas.apps.2be519f3-a56a-4801-9821-e5615bf56ec5.prod.konfluxeaas.com/maas-api' def test_bob_sre_can_mint_api_key(self, maas_api_base_url: str): """bob_sre (Site-Reliability group) can also mint an API key.""" token = _request_oidc_token(username="bob_sre", password="letmein") > data = _create_oidc_api_key(maas_api_base_url, token, name=f"e2e-bob-{uuid.uuid4().hex[:8]}") test/e2e/tests/test_external_oidc.py:315: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.2be519f3-a56a-4801-9821-e5615bf56ec5.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJxVDJDY1JhYl9pdDEzc3BOdGRCeDNHUUd2MFRYMjNiUzA0aG5xcDFyVnhBIn0.eyJle...zoWekN1tJljHMiuDlBsl2JLPbLiyaq8bHdcLEcpxqkPcbYPKM08JikfJiULQQN9H_j9t60P86XtCNMeIzZuwinQt0bKGIz_ArJZQGWsaPvG1VSv8tugfFA' name = 'e2e-bob-77b0a3ad', subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCModelAccess object at 0x7f33935d9d90> maas_api_base_url = 'https://maas.apps.2be519f3-a56a-4801-9821-e5615bf56ec5.prod.konfluxeaas.com/maas-api' def test_minted_api_key_can_list_models_and_infer(self, maas_api_base_url: str): """Complete happy path: OIDC token → API key → model list → inference.""" token = _request_oidc_token() > api_key = _create_oidc_api_key(maas_api_base_url, token)["key"] test/e2e/tests/test_external_oidc.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.2be519f3-a56a-4801-9821-e5615bf56ec5.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJxVDJDY1JhYl9pdDEzc3BOdGRCeDNHUUd2MFRYMjNiUzA0aG5xcDFyVnhBIn0.eyJle...fVoOoY_p-5dyG1JEn-WjEh45uOeJ4QF3yqi1T7VkLOUua1F3JaKbYB2Ae3jAnn_BxO_6aq3pGzn-iiisad6tiXO3K-8ZqPve_3OuEvFrmGjM22N4ZwhQ8A' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionError