/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:321: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:378: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:454: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7fa4d5be9280> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: apply_gateway_fixture(case["gateway_name"], fixture_label=case["tenant_label_name"]) bootstrap_aitenant_tenant(case) tenant = wait_for_json("tenant", TENANT_CR_NAME, case["tenant_ns"], timeout=180) assert tenant["spec"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") > wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase=("Active", "Degraded"), ) test/e2e/tests/test_multi_tenant_integration.py:77: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:253: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-sub-fc9a85bf' namespace = 'e2e-mt-fc9a85bf' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maassubscription/e2e-sub-fc9a85bf in e2e-mt-fc9a85bf did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSSubscription', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSSubscription","metadata":{"annotations":{},"name":"e2e-sub-fc9a85bf","namespace":"e2e-mt-fc9a85bf"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm","tokenRateLimits":[{"limit":100,"window":"1m"}]}],"owner":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-06-11T20:32:04Z', 'finalizers': ['maas.opendatahub.io/subscription-cleanup'], 'generation': 1, 'name': 'e2e-sub-fc9a85bf', 'namespace': 'e2e-mt-fc9a85bf', 'resourceVersion': '53577', 'uid': 'ce0842bb-0bd6-4f10-89ff-ac31ad6b8c78'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm', 'tokenRateLimits': [{'limit': 100, 'window': '1m'}]}], 'owner': {'groups': [{'name': 'system:authenticated'}]}, 'priority': 0}, 'status': {'conditions': [{'lastTransitionTime': '2026-06-11T20:32:04Z', 'message': '', 'observedGeneration': 1, 'reason': 'NoDuplicatePeers', 'status': 'False', 'type': 'SpecPriorityDuplicate'}, {'lastTransitionTime': '2026-06-11T20:32:04Z', 'message': 'failed to reconcile TokenRateLimitPolicies: model llm/facebook-opt-125m-simulated is not attached to tenant gateway for subscription e2e-mt-fc9a85bf/e2e-sub-fc9a85bf: HTTPRoute llm/facebook-opt-125m-simulated-kserve-route does not reference tenant Gateway openshift-ingress/e2e-mt-fc9a85bf', 'observedGeneration': 1, 'reason': 'ReconcileFailed', 'status': 'False', 'type': 'Ready'}], 'modelRefStatuses': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm', 'ready': True, 'reason': 'Valid'}], 'phase': 'Failed'}} test/e2e/tests/multitenancy_helpers.py:198: AssertionError/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:68: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:87: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:96: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:113: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_multi_tenant_maas_api.py:124: S24 per-tenant maas-api E2E is gated; set ENABLE_S24_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:109: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:122: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:130: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:137: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:153: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:176: S4 tenant persistence E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_subscription_isolation.py:101: S4 tenant subscription isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_subscription_isolation.py:120: S4 tenant subscription isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_rate_limit_isolation.py:154: S4 tenant rate-limit isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation lands/workspace/source/test/e2e/tests/test_tenant_rate_limit_isolation.py:163: S4 tenant rate-limit isolation E2E is gated; set ENABLE_S4_E2E=true once the backing implementation landsself = <test_external_oidc.TestOIDCTokenFlow object at 0x7fa4d60f2790> maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' def test_oidc_token_can_create_api_key(self, maas_api_base_url: str): """OIDC token from default user (alice_lead) can mint an API key.""" token = _request_oidc_token() > data = _create_oidc_api_key(maas_api_base_url, token) test/e2e/tests/test_external_oidc.py:207: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPVGstcmZtX3pocTljd3Z5enllN05tTURzZWFTcEZBZHZzV2ZlSEVpemtJIn0.eyJle...7qwVSStDZIV48UJ2kz720_C4ZMXtQZSEYkUrPmP_fgTlNn0JPR043FfL5SWnRg0gHgyGmywkBPtbngb9LAiYox6C5HMOCi5PXOq0RfY9nufnvq4miwn1SA' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCMultiUser object at 0x7fa4d60c5bb0> maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' def test_bob_sre_can_mint_api_key(self, maas_api_base_url: str): """bob_sre (Site-Reliability group) can also mint an API key.""" token = _request_oidc_token(username="bob_sre", password="letmein") > data = _create_oidc_api_key(maas_api_base_url, token, name=f"e2e-bob-{uuid.uuid4().hex[:8]}") test/e2e/tests/test_external_oidc.py:315: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPVGstcmZtX3pocTljd3Z5enllN05tTURzZWFTcEZBZHZzV2ZlSEVpemtJIn0.eyJle...GlaCM3DJfnPX017jBWldFA0Ns7WLkgCA7vk5sVcGAydc66eHCXRpXBQ7YLAbpfEyqf3B-O0V1e-DFJayTEKxk0xu8KDb6MG2evsyqAH8n50WsZR0iVuS_Q' name = 'e2e-bob-f0390b7a', subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCModelAccess object at 0x7fa4d60c5730> maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' def test_minted_api_key_can_list_models_and_infer(self, maas_api_base_url: str): """Complete happy path: OIDC token → API key → model list → inference.""" token = _request_oidc_token() > api_key = _create_oidc_api_key(maas_api_base_url, token)["key"] test/e2e/tests/test_external_oidc.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPVGstcmZtX3pocTljd3Z5enllN05tTURzZWFTcEZBZHZzV2ZlSEVpemtJIn0.eyJle...Wz3L2LNOj2-oh8RetHeckT7CEDOswoAk2xhnc_GoQNYiyIiVKAK24Ovw_7ey5smkoJRNb1l6f5BBwf8xR2l0bfUQ0c5gMEHaQRqG-389hJ0CToV1a_8a7A' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionErrorself = <test_external_oidc.TestOIDCModelAccess object at 0x7fa4d60c5580> maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' def test_revoked_api_key_cannot_access_models(self, maas_api_base_url: str): """API key minted via OIDC can be revoked and then rejected.""" token = _request_oidc_token() > key_data = _create_oidc_api_key(maas_api_base_url, token) test/e2e/tests/test_external_oidc.py:387: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.f2398922-c18b-42bb-89c9-173b7f0fb3d6.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPVGstcmZtX3pocTljd3Z5enllN05tTURzZWFTcEZBZHZzV2ZlSEVpemtJIn0.eyJle...u-Vj8ADgMcgTeDfbQBOAV5y9y-lsRaScWZAwIxea8kfJH6vozQ_65t7f6W7woJpjRAMykGi6A-vqSL8ouRcMyPf00F5Q512TcFgmaYNX0GDGlgU1rYdtbw' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token.""" body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription response = _oidc_request_with_retry( requests.post, f"{maas_api_base_url}/v1/api-keys", oidc_token, label="OIDC API key mint", json=body, ) > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 401 E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_external_oidc.py:174: AssertionError