/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1055: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7fc940575c40> def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Scale down Kuadrant controller 2. Create subscription with valid model - TRLP created but not accepted 3. Wait for subscription to enter Degraded phase (TRLP ready=false) 4. Create API key and verify inference is blocked (403 Forbidden) 5. Scale Kuadrant controller back up 6. Wait for subscription to reach Active phase (TRLP ready=true) 7. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Scale down Kuadrant controller BEFORE creating subscription log.info("Step 1: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() # Step 2: Create auth policy and subscription log.info("Step 2: Creating subscription with Kuadrant controller down...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Wait for auth policy to reconcile. In gateway-only mode, it remains Active even when # Kuadrant TRLP reconciliation is degraded. Do NOT wait for gateway AuthPolicy Enforced # here — Kuadrant is intentionally down, so Enforced can never become True # (AuthConfigs pile up "waiting … to sync"). log.info("Waiting for MaaSAuthPolicy to reconcile (Kuadrant down; skip Enforced wait)...") > _wait_for_maas_auth_policy_phase( auth_name, "Active", timeout=60, require_auth_policies=False, require_enforced=False, ) test/e2e/tests/test_subscription.py:1988: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-trlp-degraded-auth', expected_phase = 'Active' namespace = 'models-as-a-service', timeout = 60, require_auth_policies = False require_enforced = False def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=60, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: phase={phase}, authPolicies={len(auth_policies)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasauthpolicy", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, authPolicies={len(status.get('authPolicies', []))})" ) E TimeoutError: MaaSAuthPolicy e2e-trlp-degraded-auth did not reach phase 'Active' within 60s (current: phase=Pending, authPolicies=0) test/e2e/tests/test_helper.py:1134: TimeoutError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7fc940b4dc70> def test_namespace_qualified_collision_prevention(self): """1.5: Same-named CRs in two tenant namespaces use namespace-qualified TRLP tracking.""" case_a = new_discovery_case(use_default_gateway=True) case_b = new_discovery_case(use_default_gateway=True) shared_policy_name = f"e2e-shared-policy-{case_a['suffix']}" shared_sub_name = f"e2e-shared-sub-{case_a['suffix']}" case_a["policy_name"] = shared_policy_name case_a["subscription_name"] = shared_sub_name case_b["policy_name"] = shared_policy_name case_b["subscription_name"] = shared_sub_name try: for case in (case_a, case_b): apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(shared_policy_name, case["tenant_ns"]) apply_maas_subscription(shared_sub_name, case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", shared_policy_name, case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", shared_sub_name, case["tenant_ns"], FINALIZER_SUBSCRIPTION) > _wait_for_maas_auth_policy_phase(shared_policy_name, namespace=case["tenant_ns"], timeout=120) test/e2e/tests/test_tenant_namespace_discovery.py:226: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-shared-policy-1bd7d6c1', expected_phase = 'Active' namespace = 'ai-tenant-e2e-mt-1bd7d6c1', timeout = 120 require_auth_policies = False, require_enforced = True def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=60, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: phase={phase}, authPolicies={len(auth_policies)}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maasauthpolicy", name, namespace) status = cr.get("status", {}) if cr else {} > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase')}, authPolicies={len(status.get('authPolicies', []))})" ) E TimeoutError: MaaSAuthPolicy e2e-shared-policy-1bd7d6c1 did not reach phase 'Active' within 120s (current: phase=Pending, authPolicies=0) test/e2e/tests/test_helper.py:1134: TimeoutError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:353: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:183: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:212: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_discovery.py:134: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:210: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:131: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:179: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:266: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:249: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:260: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:274: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:287: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:300: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:332: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:376: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:392: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:402: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:425: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:433: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:438: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:451: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:498: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:534: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:599: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:630: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:656: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:687: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:747: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:771: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:926: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:978: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1014: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1068: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1107: EXTERNAL_OIDC not enabled