/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:321: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:378: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:454: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7fbf4b44d9a0> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: apply_gateway_fixture(case["gateway_name"], fixture_label=case["tenant_label_name"]) bootstrap_aitenant_tenant(case) tenant = wait_for_json("tenant", TENANT_CR_NAME, case["tenant_ns"], timeout=180) assert tenant["spec"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") > wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase=("Active", "Degraded"), ) test/e2e/tests/test_multi_tenant_integration.py:77: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:257: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-sub-89177eef' namespace = 'e2e-mt-89177eef' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maassubscription/e2e-sub-89177eef in e2e-mt-89177eef did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSSubscription', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSSubscription","metadata":{"annotations":{},"name":"e2e-sub-89177eef","namespace":"e2e-mt-89177eef"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm","tokenRateLimits":[{"limit":100,"window":"1m"}]}],"owner":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-06-15T03:47:31Z', 'finalizers': ['maas.opendatahub.io/subscription-cleanup'], 'generation': 1, 'name': 'e2e-sub-89177eef', 'namespace': 'e2e-mt-89177eef', 'resourceVersion': '51980', 'uid': '5c404285-30d7-42b8-9a37-5ae291c008f3'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm', 'tokenRateLimits': [{'limit': 100, 'window': '1m'}]}], 'owner': {'groups': [{'name': 'system:authenticated'}]}, 'priority': 0}, 'status': {'conditions': [{'lastTransitionTime': '2026-06-15T03:47:31Z', 'message': '', 'observedGeneration': 1, 'reason': 'NoDuplicatePeers', 'status': 'False', 'type': 'SpecPriorityDuplicate'}, {'lastTransitionTime': '2026-06-15T03:47:31Z', 'message': 'failed to reconcile TokenRateLimitPolicies: model llm/facebook-opt-125m-simulated is not attached to tenant gateway for subscription e2e-mt-89177eef/e2e-sub-89177eef: HTTPRoute llm/facebook-opt-125m-simulated-kserve-route does not reference tenant Gateway openshift-ingress/e2e-mt-89177eef', 'observedGeneration': 1, 'reason': 'ReconcileFailed', 'status': 'False', 'type': 'Ready'}], 'modelRefStatuses': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm', 'ready': True, 'reason': 'Valid'}], 'phase': 'Failed'}} test/e2e/tests/multitenancy_helpers.py:202: AssertionError/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:68: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:87: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:96: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:113: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:124: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:109: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:122: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:130: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:137: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:153: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:176: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_subscription_isolation.py:101: S4 tenant subscription isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_subscription_isolation.py:120: S4 tenant subscription isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_rate_limit_isolation.py:154: S4 tenant rate-limit isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_rate_limit_isolation.py:163: S4 tenant rate-limit isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation landsself = <test_external_oidc.TestOIDCTokenFlow object at 0x7fbf4b776e80> maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' def test_oidc_token_can_create_api_key(self, maas_api_base_url: str): """OIDC token from default user (alice_lead) can mint an API key.""" token = _request_oidc_token() > data = _create_oidc_api_key(maas_api_base_url, token) test/e2e/tests/test_external_oidc.py:207: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJLNlA0RUdQUEtmcUp2MDZfWWhpYXMyUlY0RWFRMjl3dWhMeUZyMzU5Nlo4In0.eyJle...Qx6kuP5TlpEn1rbaL6kpS3RdtOA5GZqkFA_UOhD07giL4xQEe97UFf7bmlZN9HY0StFqnxaced9ijY9XupBDm8Z2hMqVry_5PQ3_AdhTQnAeUBymwboW-A' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCMultiUser object at 0x7fbf4bc64250> maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' def test_bob_sre_can_mint_api_key(self, maas_api_base_url: str): """bob_sre (Site-Reliability group) can also mint an API key.""" token = _request_oidc_token(username="bob_sre", password="letmein") > data = _create_oidc_api_key(maas_api_base_url, token, name=f"e2e-bob-{uuid.uuid4().hex[:8]}") test/e2e/tests/test_external_oidc.py:315: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJLNlA0RUdQUEtmcUp2MDZfWWhpYXMyUlY0RWFRMjl3dWhMeUZyMzU5Nlo4In0.eyJle...RM4kSVnBNiBWojsIRpt4q5qOWZx2mb-KMk6qnnykNZbPOz78ctocaDouOAxnh1aaxfpX9R1FZQciVDBXDfqOmSCUh9sfLhY1NQA-LjRFW2fuQTIPB1SU2g' name = 'e2e-bob-0d2fb8ac', subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCModelAccess object at 0x7fbf4b444310> maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' def test_minted_api_key_can_list_models_and_infer(self, maas_api_base_url: str): """Complete happy path: OIDC token → API key → model list → inference.""" token = _request_oidc_token() > api_key = _create_oidc_api_key(maas_api_base_url, token)["key"] test/e2e/tests/test_external_oidc.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJLNlA0RUdQUEtmcUp2MDZfWWhpYXMyUlY0RWFRMjl3dWhMeUZyMzU5Nlo4In0.eyJle...EcvZCS6naCjlOZBOv7TVcm0ke5H-T54XADQK7aaVvwq2Ai6FN9I1smB7XGU8hJfVnjYR4osk7YVB-GR4sENA4zu46BG4QdA-qknNdYw9wZwvdPKgDM3Bjg' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCModelAccess object at 0x7fbf4b444160> maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' def test_revoked_api_key_cannot_access_models(self, maas_api_base_url: str): """API key minted via OIDC can be revoked and then rejected.""" token = _request_oidc_token() > key_data = _create_oidc_api_key(maas_api_base_url, token) test/e2e/tests/test_external_oidc.py:387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.4123859f-7629-442d-839d-6ec571d8b786.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJLNlA0RUdQUEtmcUp2MDZfWWhpYXMyUlY0RWFRMjl3dWhMeUZyMzU5Nlo4In0.eyJle...N2KD47r3dNuIzQChI1yETaLibmRiiz393cmsQSuKzoBDlxPKlTHOfvQbWSP4RNbTqvPV8iO1udihXZgW9kI7WrU7EKqfTUes3q63F2xMndL6XqNAR1J6gA' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionError