/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1054: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7f31169424c0> def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Scale down Kuadrant controller 2. Create subscription with valid model - TRLP created but not accepted 3. Wait for subscription to enter Degraded phase (TRLP ready=false) 4. Create API key and verify inference is blocked (403 Forbidden) 5. Scale Kuadrant controller back up 6. Wait for subscription to reach Active phase (TRLP ready=true) 7. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Scale down Kuadrant controller BEFORE creating subscription log.info("Step 1: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() # Step 2: Create auth policy and subscription log.info("Step 2: Creating subscription with Kuadrant controller down...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Wait for auth policy to reconcile. In gateway-only mode, it remains Active even when # Kuadrant TRLP reconciliation is degraded. log.info("Waiting for MaaSAuthPolicy to reconcile...") > _wait_for_maas_auth_policy_phase(auth_name, "Active", timeout=60, require_auth_policies=False) test/e2e/tests/test_subscription.py:1985: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:1046: in _wait_for_maas_auth_policy_phase _wait_for_gateway_auth_enforced(timeout=remaining) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'maas-gateway-auth', namespace = 'openshift-ingress', timeout = 60 def _wait_for_gateway_auth_enforced( name: Optional[str] = None, namespace: Optional[str] = None, timeout: int = 120, ): """Wait until the gateway Kuadrant AuthPolicy is Accepted and Enforced. MaaSAuthPolicy phase Active only means the controller reconciled CRs. HTTP calls through the gateway still need Kuadrant's AuthPolicy (typically maas-gateway-auth) to report Enforced=True; otherwise Envoy often returns an empty 403. Raises: TimeoutError: with Accepted/Enforced snapshot so failures are actionable """ name = name or GATEWAY_AUTH_POLICY_NAME namespace = namespace or GATEWAY_NAMESPACE deadline = time.time() + timeout last_snapshot = "AuthPolicy not found" log.info( "Waiting for gateway AuthPolicy %s/%s Accepted+Enforced (timeout: %ds)...", namespace, name, timeout, ) while time.time() < deadline: cr = _get_cr("authpolicy", name, namespace) if cr is None: last_snapshot = "AuthPolicy not found" else: accepted, a_reason, a_msg = _authpolicy_condition(cr, "Accepted") enforced, e_reason, e_msg = _authpolicy_condition(cr, "Enforced") last_snapshot = ( f"Accepted={accepted} reason={a_reason!r} message={a_msg!r}; " f"Enforced={enforced} reason={e_reason!r} message={e_msg!r}" ) if accepted == "True" and enforced == "True": log.info("Gateway AuthPolicy %s/%s is Accepted and Enforced", namespace, name) return cr log.debug("Gateway AuthPolicy %s/%s not ready: %s", namespace, name, last_snapshot) time.sleep(2) > raise TimeoutError( f"Gateway AuthPolicy {namespace}/{name} was not Accepted+Enforced within {timeout}s " f"(last status: {last_snapshot}). Empty HTTP 403 from maas-api usually means Kuadrant " f"has not finished enforcing auth on the gateway after MaaSAuthPolicy changes." ) E TimeoutError: Gateway AuthPolicy openshift-ingress/maas-gateway-auth was not Accepted+Enforced within 60s (last status: Accepted=True reason='Accepted' message='AuthPolicy has been accepted'; Enforced=False reason='Unknown' message='AuthPolicy waiting for the following components to sync: [AuthConfig (7371c34ce8e4df2309ee8f952c87f921947b289427b6e9ea579dcb9970fc1b86) AuthConfig (bf72a6316f6ed79299511e58d068836cdc71dbca5e23944f783c9340ffa0aee1) AuthConfig (630f18b14e5ce0234d4da773011770ac52ec305a1af3397f72696bf5262056bb) AuthConfig (d66f5b12bbb76f83ef1cb89313ae7356d9a4c974f276c1405eefa0cb8d66ceda) AuthConfig (30c4b39450ec876cfb2b0e66cbea2a74b7bcfafcf83d80a94c503487492e5a48) AuthConfig (311b1be286674fd5684c9ac59b318287dade9769cfe4aeebd8c88e2dc6b72418) AuthConfig (1ef3f8313d73a7c14b76ea704cc3f23f015b2ab50034887cf570208a050e9863) AuthConfig (b64d1d3a1a41de71c5b05aa58c2da5025409dfbb29fbaa63fe6a7b8fb71bf42e) AuthConfig (a124e7db3701742524b9534a4ba1eb68cc6d8f337ccc26c3f8f17b958cc795d0) AuthConfig (98b96558d098c10ea5cd4b877712861a71c554909b27785cf9b672cf28954a5a) AuthConfig (86cbb62fb4d82d4dc402b3281444539a5625c4bb4c86bbc4912c70e690a2e374) AuthConfig (4dc577fd60594d78a4a8bebe396f4b5a928f41bdc3f95c06d717cf1ddc3158b2) AuthConfig (06b05a8e4dd69bdc4f333ad9509ee8cad3c5c0f5dd48edbe94c1158b60f79ed3) AuthConfig (a394737442ca47bfbcb906bcf34d05c5515587984942470db6d6858608ab543a) AuthConfig (18e32965997cdd9967355c6fa5264ed12c0a215989d459ed88d7d6de02865f76) AuthConfig (e38d76c6f386f12bc12190c87b39e6e77e182be454f85659a9197c301f2cd9be) AuthConfig (cc09b530b46a73b0d4ddb40e465580cff15db19d77e93e4903c9737647deeb1a) AuthConfig (a7740f6e77d024a437502c023fb1d8939cb3cc2b9ed7f4073af7d779e19459eb) AuthConfig (3c0d47082320c9bb02d3788ab2b052c696af91abc2ae438437ae71a26936c7c2) AuthConfig (5fea747cb803a5ee3aeb620187bd9ec74ccccd10a92474ef528215a7ff146c8f) AuthConfig (ba8750633b915855593e941d13c912f4f6c6d6c6a659c241b67d178bbe85f188) AuthConfig (98dd9311f361e75de96f77b800598c9f5c79c7fa539cfd493b5f7631f8efab90) AuthConfig (4375a0165d777c3dbcac3b90772aafb3ba82b742e01878423b9fb1e122337433) AuthConfig (9903213c635804dd416e9f12956f0fa896195627091daaf593a30df64cf640c5) AuthConfig (e9607d3c9394ff787a7cb9306c470b1007979008e27c6d6c68281553a4653faa) AuthConfig (305bde7d554d0d2810b101d2b8d4e0e15521c2afbf16166555b8e9aa4ed39a3d) AuthConfig (d3b195a61a7f24c6bf1fba40f9f2e2565facb6af92e959c1546ac398a9172618) AuthConfig (9f3df6834dcf2df14d188decd66432f3940127e3a8261065cd3a44bd9ed1db6d) AuthConfig (35d34d59676c333235d7c9f02273e0380bb39f27cfd30856fedc0f7c0e5f79aa) AuthConfig (452d598a0ad1047944cd02d3654e6793847781ecc87763544a69e05020a86cf3) AuthConfig (3d32277ea538e86091b57e14fc6f6582cbe1daa3e7e81223282a75b03a512083) AuthConfig (a89b0896df0d8cd430f1c81b6eb292ddc044daed393537d009a6330718f58d4b) AuthConfig (e50f5fdcb9fb7e124557afe69edae9a95d05da488eae1cc4b5c1c7c1220a826a) AuthConfig (3774b789b89cb794db9f2ae5a4e410c6a000c480af86f9e4466977b4dc7de25a) AuthConfig (3dc75fc8307b952a3e873400cf417e90f2861e8a225abec4b22708deb7901db7) AuthConfig (c098209e055107073c6ded1d63e221315c2239f3cf17a7d15056a7494c5714fb) AuthConfig (f9a2a1e13eaadb4f450e63a68a3d05ec02a537019380f832f5992d1c5cdee533) AuthConfig (47d4519f979dd8b06fb3b12ecf63e079887e8b64c789a1ff79b4c429f3a6aa2a) AuthConfig (df733b2c652bfe2458c9e19932b1091e939c1a62178d6879462b3b4f73fca4bb) AuthConfig (cd452b3491b4064e772f38d1a288406bb3889f19efd54af26f6554f07046a8c0) AuthConfig (707878bcd632125ecab845de29fc41570ff2eb882a04dea05a2e77c0ea6d31f9) AuthConfig (331dc257e65f55f80700a586f7807093eda7b3d8e7d91215dc3c47731508480c) AuthConfig (888fdad5bc63ac36546012ce97b2af4b7b1a45ef243fd2eba72f12c306f06e3e) AuthConfig (4325b2a5e55bf181d1f503f5c3cbce3259c00ca301216d4ea5bc6b962627f1bd) AuthConfig (3dd75e19cd66d310c30638e330078972afd6d2d96305f91055bc6a6f363fb8d3) AuthConfig (01918e70442d2cb18c94316131b0e0a241b4a5d12df5b1e1981a3bef420bbdb5) AuthConfig (edcceb5a2e0cf1edde0fc3ed43068ce5b123a6fdc41949959c2c3b7a5a48bf24) AuthConfig (319a353672fc7601c875fa3f8b807adf60b64f093f022b2a1c3dfef3ac8cd4f4) AuthConfig (5c6918972d1a7f8ad5201880b7d6d9c1191ca1f25376682f691def7868e3fd02) AuthConfig (9bbfc0b2e9e9acadd41342cd7c36a24afa8a5eef942d161782a6adae8411158a) AuthConfig (80f10756c0e833d16937036ed66f1daf5bef95559ef05a5e852766b97b9bdaef) AuthConfig (101d9a4d9b715268b3a3623fb114473501068b7352d534f324f4a3121b535746)]'). Empty HTTP 403 from maas-api usually means Kuadrant has not finished enforcing auth on the gateway after MaaSAuthPolicy changes. test/e2e/tests/test_helper.py:848: TimeoutErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f3116f37d00> def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid model (will be Active) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Verify it starts as Active cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") log.info(f"Initial phase: {phase}") assert phase == "Active", f"Expected Active initially, got {phase}" # Create API key while Active api_key = _create_api_key( oc_token, name="failed-sub-test", subscription=subscription_name ) # Verify inference works while Active log.info("Testing inference while Active...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) assert r.status_code == 200, f"Expected 200 while Active, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference works with Active subscription") # Manually patch subscription to Failed phase import subprocess import json from datetime import datetime log.info("Manually patching subscription to Failed phase...") patch_data = { "status": { "phase": "Failed", "conditions": [ { "type": "Ready", "status": "False", "reason": "Failed", "message": "Subscription failed", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") } ], "modelRefStatuses": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Model failed" } ] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch to Failed phase: {result.stderr}" # Verify phase is Failed cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") > assert phase == "Failed", f"Expected Failed phase after patch, got {phase}" E AssertionError: Expected Failed phase after patch, got Active E assert 'Active' == 'Failed' E E - Failed E + Active test/e2e/tests/test_subscription.py:2355: AssertionError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:353: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:209: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_discovery.py:134: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:210: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:131: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:179: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:266: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:249: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:260: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:274: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:287: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:300: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:332: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:376: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:392: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:402: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:425: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:433: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:438: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:451: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:498: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:534: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:599: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:630: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:656: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:687: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:747: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:771: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:926: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:978: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1014: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1068: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1107: EXTERNAL_OIDC not enabled