/workspace/source/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gatewayself = <test_models_endpoint.TestModelsEndpoint object at 0x7f3aa6d19cd0> def test_api_key_scoped_to_subscription(self): """ Test: API key returns only models from its bound subscription. API keys are scoped to a specific subscription at mint time. The gateway automatically injects X-MaaS-Subscription from the key's subscription. Expected: HTTP 200 with models only from the key's subscription, even if the user has access to multiple subscriptions. """ ns = _ns() auth_policy_name = "e2e-api-key-scoped-auth" subscription_name = "e2e-api-key-scoped-sub" sa_name = "e2e-api-key-scoped-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_subscription_discovery_ready(subscription_name, namespace=ns) # Create API key bound to subscription_name api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) > _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) test/e2e/tests/test_models_endpoint.py:1558: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-api-key-scoped-auth', expected_phase = 'Active' namespace = 'ai-tenant-e2e-worker-w5-743a5e', timeout = 120 require_auth_policies = False, require_enforced = False def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=AUTHPOLICY_PHASE_TIMEOUT, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") last_snapshot = "not found" while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr is None: last_snapshot = "not found" else: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) ready_count = sum(1 for ap in auth_policies if ap.get("ready") is True) last_snapshot = f"phase={phase}, authPolicies={len(auth_policies)} (ready={ready_count})" if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: {last_snapshot}") time.sleep(2) # Report what the loop last saw: a re-read after the deadline can show a # state the loop never evaluated. > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(last seen: {last_snapshot})" ) E TimeoutError: MaaSAuthPolicy e2e-api-key-scoped-auth did not reach phase 'Active' within 120s (last seen: phase=Pending, authPolicies=0 (ready=0)) test/e2e/tests/test_helper.py:1889: TimeoutErrorself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f595ede3dd0> def test_default_tenant_unaffected_by_multitenancy_enablement(self): """7.2: Default tenant namespace still reconciles while discovery mode is enabled.""" ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-default-int-auth-{suffix}" subscription_name = f"e2e-default-int-sub-{suffix}" try: apply_maas_auth_policy(policy_name, ns) apply_maas_subscription(subscription_name, ns) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/e2e/tests/test_multi_tenant_integration.py:213: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:324: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-int-auth-ba9746' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f595ea01260> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-int-auth-ba9746 in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-int-auth-ba9746","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-29T22:01:19Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-int-auth-ba9746', 'namespace': 'models-as-a-service', 'resourceVersion': '45377', 'uid': 'daf1a2fd-090c-4b5b-8959-8d36ae2d88f1'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-29T22:02:00Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 22', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/e2e/tests/multitenancy_helpers.py:269: AssertionErrorself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f4d46767250> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) > wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") test/e2e/tests/test_multi_tenant_integration.py:116: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:324: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-751a7416' namespace = 'ai-tenant-e2e-mt-751a7416' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f4d465879c0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-751a7416 in ai-tenant-e2e-mt-751a7416 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-751a7416","namespace":"ai-tenant-e2e-mt-751a7416"},"spec":{"modelRefs":[{"name":"e2e-lifecycle-model-751a7416","namespace":"ai-tenant-e2e-mt-751a7416"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-29T22:03:07Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-751a7416', 'namespace': 'ai-tenant-e2e-mt-751a7416', 'resourceVersion': '49793', 'uid': '1cc1b0c6-7f9f-4017-acd5-757854ec0963'}, 'spec': {'modelRefs': [{'name': 'e2e-lifecycle-model-751a7416', 'namespace': 'ai-tenant-e2e-mt-751a7416'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-29T22:03:07Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-mt-751a7416-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (33af6309c27b42b051d55f552a0d52d387b8e7f51f6667192830fa6dc7950bf4) AuthConfig (1234dfaa12906aac474683cd8fc2901109eb17347a54271a14da2aada24449b1) AuthConfig (bcf0630d9c1004a6ccd67bc3ac798a61e4d1960c8c849d0e5ee871eec7737206) AuthConfig (6976ad314850b75984999b5f33949b4bcc43cffda559cf22dd09d3a4873eab81) AuthConfig (a968152bb9225c9ea35f36214474f32d75670ada2cf93f1db39faa7c3a1f273d) AuthConfig (f1c35998436af30143d3b46fd0e4a77e08c24835420cf6b6a19efaa5f5d26017) AuthConfig (0452b3e4dbb020bac4153406e9862f0580742572a793cfa7f5f722040e6f411d) AuthConfig (dd33910ebe596c0d28ded0bd9a5e8d4e8e3f947b79cca643687d2a3392b441a4) AuthConfig (59aae3bf20375ce20f479973deb63273a5a7052939b38923eb70176eea9f4f23) AuthConfig (8d26735f5a2d3033b0cf75494e48753db7815d9fc7118a96ee6cd1d64f7e4a90) AuthConfig (c4aef70da371b116197f21c16a451c890970e9bf49a41934a8440a868013e84f) AuthConfig (370a864daf986626844b8835229c877dc765f11b84553353b4d17429ab8dd54c) AuthConfig (d3d98c1c7916ae0ca0902eca1a9351bacabc953324d740afbf642c97eb8df1a0) AuthConfig (0c2b8456150c9f6813697bbea848fa98f895eb612876529a3223e7ba30559c7a) AuthConfig (5b19905cabb86f740af5c5d0b959fcb7194dbaa45a81dd9bbe5371d1ddf949ce) AuthConfig (fb177c0aef047432cb71acacea7f4dffad2e8c773a5ac6feac28545a651cc1d5) AuthConfig (ebb538b4c2f28f2f245b7c7c89d7bda3dbff0da7d39ce0d3f8c90103bc5751a4) AuthConfig (1de463cf9bd963cf32f579c9b01c90fddceeab0a4795072261a5640fd279deb7)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/e2e/tests/multitenancy_helpers.py:269: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f1014fd39d0> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_subscription_discovery_ready(sub1_name, namespace=maas_ns) _wait_for_subscription_discovery_ready(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" > _wait_for_maas_auth_policy_phase(auth1_name, require_enforced=False) test/e2e/tests/test_models_endpoint.py:2004: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-multi-keys-auth1', expected_phase = 'Active' namespace = 'ai-tenant-e2e-worker-w0-e4e24e', timeout = 120 require_auth_policies = False, require_enforced = False def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=AUTHPOLICY_PHASE_TIMEOUT, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") last_snapshot = "not found" while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr is None: last_snapshot = "not found" else: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) ready_count = sum(1 for ap in auth_policies if ap.get("ready") is True) last_snapshot = f"phase={phase}, authPolicies={len(auth_policies)} (ready={ready_count})" if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: {last_snapshot}") time.sleep(2) # Report what the loop last saw: a re-read after the deadline can show a # state the loop never evaluated. > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(last seen: {last_snapshot})" ) E TimeoutError: MaaSAuthPolicy e2e-multi-keys-auth1 did not reach phase 'Active' within 120s (last seen: phase=Pending, authPolicies=0 (ready=0)) test/e2e/tests/test_helper.py:1889: TimeoutError/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_networkpolicy.py:119: AIGC praxis NetworkPolicy does not include OTLP 4317 egress (MaaS Go IPP bundle only)self = <test_negative_security.TestMissingModelRef object at 0x7f9e83f1b5d0> def test_authpolicy_with_nonexistent_model_ref(self): """MaaSAuthPolicy does not generate per-model AuthPolicies in gateway-only mode. Creates an auth policy referencing one valid model and one ghost model, waits for Degraded phase, then asserts that no legacy per-model AuthPolicies exist for either model. """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-ghost-policy-{suffix}" ghost_model = f"nonexistent-model-{suffix}" try: _create_test_auth_policy( policy_name, [MODEL_REF, ghost_model], groups=["system:authenticated"], ) > _wait_for_maas_auth_policy_phase(policy_name, "Degraded", timeout=60, require_auth_policies=False) test/e2e/tests/test_negative_security.py:578: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-neg-ghost-policy-df60e057', expected_phase = 'Degraded' namespace = 'models-as-a-service', timeout = 60, require_auth_policies = False require_enforced = True def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=AUTHPOLICY_PHASE_TIMEOUT, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") last_snapshot = "not found" while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr is None: last_snapshot = "not found" else: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) ready_count = sum(1 for ap in auth_policies if ap.get("ready") is True) last_snapshot = f"phase={phase}, authPolicies={len(auth_policies)} (ready={ready_count})" if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: {last_snapshot}") time.sleep(2) # Report what the loop last saw: a re-read after the deadline can show a # state the loop never evaluated. > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(last seen: {last_snapshot})" ) E TimeoutError: MaaSAuthPolicy e2e-neg-ghost-policy-df60e057 did not reach phase 'Degraded' within 60s (last seen: phase=Pending, authPolicies=0 (ready=0)) test/e2e/tests/test_helper.py:1889: TimeoutError/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)