/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1055: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7f3c1c5f8490> def test_namespace_qualified_collision_prevention(self): """1.5: Same-named CRs in two tenant namespaces use namespace-qualified TRLP tracking.""" case_a = new_discovery_case() case_b = new_discovery_case() shared_policy_name = f"e2e-shared-policy-{case_a['suffix']}" shared_sub_name = f"e2e-shared-sub-{case_a['suffix']}" case_a["policy_name"] = shared_policy_name case_a["subscription_name"] = shared_sub_name case_b["policy_name"] = shared_policy_name case_b["subscription_name"] = shared_sub_name try: for case in (case_a, case_b): bootstrap_aitenant_tenant(case) apply_maas_auth_policy(shared_policy_name, case["tenant_ns"]) apply_maas_subscription(shared_sub_name, case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", shared_policy_name, case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", shared_sub_name, case["tenant_ns"], FINALIZER_SUBSCRIPTION) _wait_for_maas_auth_policy_phase(shared_policy_name, namespace=case["tenant_ns"], timeout=120, require_enforced=False) assert_no_per_model_authpolicy(MODEL_REF, MODEL_NAMESPACE) expected_a_sub = f"{case_a['tenant_ns']}/{shared_sub_name}" expected_b_sub = f"{case_b['tenant_ns']}/{shared_sub_name}" > sub_contributors = wait_for_annotation_contains( "tokenratelimitpolicy", f"maas-trlp-{MODEL_REF}", MODEL_NAMESPACE, "maas.opendatahub.io/subscriptions", [expected_a_sub, expected_b_sub], ) test/e2e/tests/test_tenant_namespace_discovery.py:232: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:358: in wait_for_annotation_contains wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'tokenratelimitpolicy', name = 'maas-trlp-facebook-opt-125m-simulated' namespace = 'llm' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: tokenratelimitpolicy/maas-trlp-facebook-opt-125m-simulated in llm did not satisfy condition. Last object: {'apiVersion': 'kuadrant.io/v1alpha1', 'kind': 'TokenRateLimitPolicy', 'metadata': {'annotations': {'maas.opendatahub.io/subscriptions': 'models-as-a-service/simulator-subscription'}, 'creationTimestamp': '2026-07-29T21:10:32Z', 'generation': 19, 'labels': {'app.kubernetes.io/component': 'token-rate-limit-policy', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'maas-subscription', 'maas.opendatahub.io/model': 'facebook-opt-125m-simulated', 'maas.opendatahub.io/model-namespace': 'llm'}, 'name': 'maas-trlp-facebook-opt-125m-simulated', 'namespace': 'llm', 'ownerReferences': [{'apiVersion': 'gateway.networking.k8s.io/v1', 'blockOwnerDeletion': True, 'controller': True, 'kind': 'HTTPRoute', 'name': 'facebook-opt-125m-simulated-kserve-route', 'uid': '6de99e50-c32d-4081-933a-d906ad35ccb9'}], 'resourceVersion': '50469', 'uid': '18d0a3c4-88b1-40a3-a2cb-d332eb064766'}, 'spec': {'limits': {'models-as-a-service-simulator-subscription-facebook-opt-125m-simulated-tokens': {'counters': [{'expression': 'auth.identity.userid'}], 'rates': [{'limit': 100, 'window': '1m'}], 'when': [{'predicate': 'auth.identity.selected_subscription_key == "models-as-a-service/simulator-subscription@llm/facebook-opt-125m-simulated" && !request.path.endsWith("/v1/models")'}]}}, 'targetRef': {'group': 'gateway.networking.k8s.io', 'kind': 'HTTPRoute', 'name': 'facebook-opt-125m-simulated-kserve-route'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-07-29T21:10:37Z', 'message': 'TokenRateLimitPolicy has been accepted', 'reason': 'Accepted', 'status': 'True', 'type': 'Accepted'}, {'lastTransitionTime': '2026-07-29T21:21:32Z', 'message': 'TokenRateLimitPolicy has been successfully enforced', 'reason': 'Enforced', 'status': 'True', 'type': 'Enforced'}], 'observedGeneration': 19}} test/e2e/tests/multitenancy_helpers.py:227: AssertionError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:352: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:183: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f3c1c30ed30> def test_same_named_resources_across_tenants(self): """7.3: Same-named MaaS resources in separate tenant namespaces both contribute safely.""" case_a = new_discovery_case() case_b = new_discovery_case() shared_policy = f"e2e-shared-int-policy-{case_a['suffix']}" shared_sub = f"e2e-shared-int-sub-{case_a['suffix']}" for case in (case_a, case_b): case["policy_name"] = shared_policy case["subscription_name"] = shared_sub try: for case in (case_a, case_b): bootstrap_aitenant_tenant(case) apply_maas_auth_policy(shared_policy, case["tenant_ns"]) apply_maas_subscription(shared_sub, case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", shared_policy, case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", shared_sub, case["tenant_ns"], FINALIZER_SUBSCRIPTION) wait_for_status_phase("maasauthpolicy", shared_policy, case["tenant_ns"], expected_phase="Active") expected_subs = [f"{case_a['tenant_ns']}/{shared_sub}", f"{case_b['tenant_ns']}/{shared_sub}"] > wait_for_annotation_contains( "tokenratelimitpolicy", f"maas-trlp-{MODEL_REF}", MODEL_NAMESPACE, "maas.opendatahub.io/subscriptions", expected_subs, ) test/e2e/tests/test_multi_tenant_integration.py:188: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:358: in wait_for_annotation_contains wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'tokenratelimitpolicy', name = 'maas-trlp-facebook-opt-125m-simulated' namespace = 'llm' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: tokenratelimitpolicy/maas-trlp-facebook-opt-125m-simulated in llm did not satisfy condition. Last object: {'apiVersion': 'kuadrant.io/v1alpha1', 'kind': 'TokenRateLimitPolicy', 'metadata': {'annotations': {'maas.opendatahub.io/subscriptions': 'models-as-a-service/simulator-subscription'}, 'creationTimestamp': '2026-07-29T21:10:32Z', 'generation': 23, 'labels': {'app.kubernetes.io/component': 'token-rate-limit-policy', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'maas-subscription', 'maas.opendatahub.io/model': 'facebook-opt-125m-simulated', 'maas.opendatahub.io/model-namespace': 'llm'}, 'name': 'maas-trlp-facebook-opt-125m-simulated', 'namespace': 'llm', 'ownerReferences': [{'apiVersion': 'gateway.networking.k8s.io/v1', 'blockOwnerDeletion': True, 'controller': True, 'kind': 'HTTPRoute', 'name': 'facebook-opt-125m-simulated-kserve-route', 'uid': '6de99e50-c32d-4081-933a-d906ad35ccb9'}], 'resourceVersion': '63487', 'uid': '18d0a3c4-88b1-40a3-a2cb-d332eb064766'}, 'spec': {'limits': {'models-as-a-service-simulator-subscription-facebook-opt-125m-simulated-tokens': {'counters': [{'expression': 'auth.identity.userid'}], 'rates': [{'limit': 100, 'window': '1m'}], 'when': [{'predicate': 'auth.identity.selected_subscription_key == "models-as-a-service/simulator-subscription@llm/facebook-opt-125m-simulated" && !request.path.endsWith("/v1/models")'}]}}, 'targetRef': {'group': 'gateway.networking.k8s.io', 'kind': 'HTTPRoute', 'name': 'facebook-opt-125m-simulated-kserve-route'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-07-29T21:10:37Z', 'message': 'TokenRateLimitPolicy has been accepted', 'reason': 'Accepted', 'status': 'True', 'type': 'Accepted'}, {'lastTransitionTime': '2026-07-29T21:36:37Z', 'message': 'TokenRateLimitPolicy has been successfully enforced', 'reason': 'Enforced', 'status': 'True', 'type': 'Enforced'}], 'observedGeneration': 23}} test/e2e/tests/multitenancy_helpers.py:227: AssertionError/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:212: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_Bself = <test_tenant_subscription_isolation.TestTenantSubscriptionIsolation object at 0x7f3c1c6224c0> tenant_subscriptions = {'auth_policy': 'e2e-sub-iso-auth-daeb46', 'shared': 'e2e-shared-sub-daeb46', 'tenant_a': {'base_url': 'https://e2e-sh...est-model-b69e52', 'model_namespace': 'ai-tenant-e2e-shared-a-b69e52', ...}, 'tenant_a_only': 'e2e-a-only-daeb46', ...} def test_subscription_selection_per_tenant(self, tenant_subscriptions): """4.2: Same-named subscriptions resolve to the namespace behind each tenant endpoint.""" shared = tenant_subscriptions["shared"] tenant_a = tenant_subscriptions["tenant_a"] tenant_b = tenant_subscriptions["tenant_b"] > key_a = _create_key_for_subscription(tenant_a, shared) test/e2e/tests/test_tenant_subscription_isolation.py:185: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ tenant = {'base_url': 'https://e2e-shared-a-b69e52.apps.fbdc818f-be73-44da-8635-be9c76a2e1db.prod.konfluxeaas.com/maas-api', 'g... 'e2e-shared-a-b69e52', 'model_name': 'sub-test-model-b69e52', 'model_namespace': 'ai-tenant-e2e-shared-a-b69e52', ...} subscription = 'e2e-shared-sub-daeb46' def _create_key_for_subscription(tenant: dict[str, str], subscription: str) -> str: response = create_api_key_at( tenant["base_url"], _get_cluster_token(), f"e2e-sub-iso-{uuid.uuid4().hex[:6]}", subscription=subscription, ) > assert response.status_code in (200, 201), ( f"create API key for {tenant['name']} failed: {response_summary(response)}" ) E AssertionError: create API key for e2e-shared-a-b69e52 failed: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_subscription_isolation.py:152: AssertionErrortenant_env = ({'backend_model_name': 'facebook/opt-125m', 'base_url': 'https://e2e-shared-a-b69e52.apps.fbdc818f-be73-44da-8635-be9...db.prod.konfluxeaas.com/maas-api', 'gateway_name': 'e2e-shared-b-ac47f7', 'model_name': 'rate-test-model-ac47f7', ...}) @pytest.fixture def tenant_rate_limit_setup(tenant_env): suffix = uuid.uuid4().hex[:6] tenant_a, tenant_b = tenant_env policy_name = f"e2e-rate-iso-auth-{suffix}" sub_a = f"e2e-rate-iso-a-{suffix}" sub_b = f"e2e-rate-iso-b-{suffix}" try: for tenant in tenant_env: apply_maas_auth_policy( policy_name, tenant["namespace"], model_ref=tenant["model_name"], model_namespace=tenant["model_namespace"], ) wait_for_status_phase( "maasauthpolicy", policy_name, tenant["namespace"], expected_phase="Active", ) for tenant in (tenant_a, tenant_b): wait_for_gateway_authpolicy_ready(tenant["gateway_name"]) apply_maas_subscription( sub_a, tenant_a["namespace"], model_ref=tenant_a["model_name"], model_namespace=tenant_a["model_namespace"], token_limit=3, window="1m", ) apply_maas_subscription( sub_b, tenant_b["namespace"], model_ref=tenant_b["model_name"], model_namespace=tenant_b["model_namespace"], token_limit=100, window="1m", ) for name, namespace in ((sub_a, tenant_a["namespace"]), (sub_b, tenant_b["namespace"])): wait_for_status_phase( "maassubscription", name, namespace, expected_phase=("Active", "Degraded"), ) _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=namespace, timeout=120, ) for tenant in (tenant_a, tenant_b): wait_for_status_phase( "maasmodelref", tenant["model_name"], tenant["namespace"], expected_phase="Ready", timeout=180, ) oc_token = _get_cluster_token() key_a_response = create_api_key_at( tenant_a["base_url"], oc_token, f"e2e-rate-a-{suffix}", subscription=sub_a, ) > assert key_a_response.status_code in (200, 201), response_summary(key_a_response) E AssertionError: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_rate_limit_isolation.py:128: AssertionErrortenant_env = ({'backend_model_name': 'facebook/opt-125m', 'base_url': 'https://e2e-shared-a-b69e52.apps.fbdc818f-be73-44da-8635-be9...db.prod.konfluxeaas.com/maas-api', 'gateway_name': 'e2e-shared-b-ac47f7', 'model_name': 'rate-test-model-ac47f7', ...}) @pytest.fixture def tenant_rate_limit_setup(tenant_env): suffix = uuid.uuid4().hex[:6] tenant_a, tenant_b = tenant_env policy_name = f"e2e-rate-iso-auth-{suffix}" sub_a = f"e2e-rate-iso-a-{suffix}" sub_b = f"e2e-rate-iso-b-{suffix}" try: for tenant in tenant_env: apply_maas_auth_policy( policy_name, tenant["namespace"], model_ref=tenant["model_name"], model_namespace=tenant["model_namespace"], ) wait_for_status_phase( "maasauthpolicy", policy_name, tenant["namespace"], expected_phase="Active", ) for tenant in (tenant_a, tenant_b): wait_for_gateway_authpolicy_ready(tenant["gateway_name"]) apply_maas_subscription( sub_a, tenant_a["namespace"], model_ref=tenant_a["model_name"], model_namespace=tenant_a["model_namespace"], token_limit=3, window="1m", ) apply_maas_subscription( sub_b, tenant_b["namespace"], model_ref=tenant_b["model_name"], model_namespace=tenant_b["model_namespace"], token_limit=100, window="1m", ) for name, namespace in ((sub_a, tenant_a["namespace"]), (sub_b, tenant_b["namespace"])): wait_for_status_phase( "maassubscription", name, namespace, expected_phase=("Active", "Degraded"), ) _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=namespace, timeout=120, ) for tenant in (tenant_a, tenant_b): wait_for_status_phase( "maasmodelref", tenant["model_name"], tenant["namespace"], expected_phase="Ready", timeout=180, ) oc_token = _get_cluster_token() key_a_response = create_api_key_at( tenant_a["base_url"], oc_token, f"e2e-rate-a-{suffix}", subscription=sub_a, ) > assert key_a_response.status_code in (200, 201), response_summary(key_a_response) E AssertionError: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_rate_limit_isolation.py:128: AssertionError