/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1054: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7fb71a2f3130> def test_different_modelrefs_same_model_id(self): """ Test 7: Different modelRefs serving same model ID return separate entries. Creates two ephemeral LLMInferenceServices / MaaSModelRefs that both serve the same unique ID (test/e2e-same-model-id). Fixture models intentionally use distinct served IDs, so this case is covered with short-lived CRs. Under BBR, both modelRefs share the gateway base URL. Deduplication keys on (model ID, URL, owned_by), so distinct MaaSModelRefs still appear as 2 entries distinguished by owned_by (namespace/name), not by URL. """ log.info("Test 7: Different modelRefs same ID remain separate via owned_by (BBR)") sa_name = "e2e-models-diff-refs-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-diff-refs-subscription" auth_policy_name = "e2e-diff-refs-auth" suffix = uuid.uuid4().hex[:8] shared_served_id = "test/e2e-same-model-id" expected_canonical_id = f"publishers/{MODEL_NAMESPACE}/models/{shared_served_id}" model_ref_a = f"e2e-same-id-a-{suffix}" model_ref_b = f"e2e-same-id-b-{suffix}" expected_owned_by = { f"{MODEL_NAMESPACE}/{model_ref_a}", f"{MODEL_NAMESPACE}/{model_ref_b}", } api_key = None try: # Create two backends that advertise the same served model ID for ref in (model_ref_a, model_ref_b): _create_llmis( ref, MODEL_NAMESPACE, "maas-default-gateway", GATEWAY_NAMESPACE, model_name=shared_served_id, ) _create_maas_model_ref(ref, MODEL_NAMESPACE, ref) sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) log.info(f"Creating auth policy with {model_ref_a} and {model_ref_b}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": model_ref_a, "namespace": MODEL_NAMESPACE}, {"name": model_ref_b, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) log.info(f"Creating subscription with {model_ref_a} and {model_ref_b}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": model_ref_a, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": model_ref_b, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 200, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) _wait_for_maas_auth_policy_phase( auth_policy_name, namespace=maas_ns, timeout=120, ) _wait_for_maas_subscription_phase( subscription_name, namespace=maas_ns, timeout=180, require_model_statuses=True, ) _wait_for_model_ready(model_ref_a, namespace=MODEL_NAMESPACE, timeout=180) _wait_for_model_ready(model_ref_b, namespace=MODEL_NAMESPACE, timeout=180) api_key = _create_api_key(sa_token, name="e2e-diff-refs-test-key", subscription=subscription_name) _wait_reconcile() log.info(f"Querying /v1/models with subscription: {subscription_name}") request_headers = { "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, } deadline = time.time() + 120 r = None models = [] model_ids = [] while time.time() < deadline: r = _get_models_with_gateway_retry(headers=request_headers) if r.status_code == 200: models = r.json().get("data") or [] assert isinstance(models, list), "Models should be a list" model_ids = [m["id"] for m in models] owned_bys = { m.get("owned_by") for m in models if m.get("id") == expected_canonical_id and m.get("owned_by") } if len(models) == 2 and owned_bys == expected_owned_by: break log.info( "Models response not fully propagated yet; " f"got {len(models)} entries: {model_ids}, owned_by={owned_bys}" ) else: log.info(f"Waiting for /v1/models HTTP 200; got {r.status_code}: {r.text[:200]}") time.sleep(5) assert r is not None, "Expected /v1/models response, got none" assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" assert isinstance(models, list), "Models should be a list" unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 different modelRefs both serving '{shared_served_id}'") assert len(unique_ids) == 1, \ f"Expected only 1 unique model ID (both modelRefs serve {expected_canonical_id}), got {len(unique_ids)}: {unique_ids}" assert expected_canonical_id in unique_ids, \ f"Expected to find '{expected_canonical_id}', but got {unique_ids}" assert len(models) == 2, \ f"Expected 2 entries (different owned_by), got {len(models)}: {json.dumps(models, indent=2)}" urls = [m["url"] for m in models if "url" in m] assert len(urls) == 2, f"Expected 2 URLs, got {len(urls)}" > assert urls[0] == urls[1], \ f"Under BBR both modelRefs should share the gateway base URL, got: {urls}" E AssertionError: Under BBR both modelRefs should share the gateway base URL, got: ['https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/e2e-same-id-a-57f86f6c', 'https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/e2e-same-id-b-57f86f6c'] E assert 'https://maas...id-a-57f86f6c' == 'https://maas...id-b-57f86f6c' E E - https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/e2e-same-id-b-57f86f6c E ? ^ E + https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/e2e-same-id-a-57f86f6c E ? ^ test/e2e/tests/test_models_endpoint.py:901: AssertionError/workspace/source/test/e2e/tests/test_external_models.py:219: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:224: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:229: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:240: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:253: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:269: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:298: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:351: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:394: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_external_models.py:409: External endpoint httpbin.org is not reachable (disconnected environment?)/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not assertedself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7fb71a22ae80> def test_aitenant_delete_cleans_maas_resources_and_preserves_user_objects(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) _apply_aitenant(case) _assert_aitenant_bootstrap_resources(case) _apply( { "apiVersion": "v1", "kind": "Secret", "metadata": {"name": case["user_secret"], "namespace": case["tenant_ns"]}, "type": "Opaque", "stringData": {"purpose": "unrelated-user-content"}, } ) _apply( { "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", "metadata": {"name": case["user_rolebinding"], "namespace": case["tenant_ns"]}, "roleRef": { "apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": case["tenant_admin_role"], }, "subjects": [ { "apiGroup": "rbac.authorization.k8s.io", "kind": "User", "name": "e2e-preserved-user", } ], } ) > _delete_aitenant(case) test/e2e/tests/test_aitenant_lifecycle.py:584: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:253: in _delete_aitenant _delete(AITENANT_KIND, case["aitenant_name"], AITENANT_NAMESPACE, timeout="180s") _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ait-ed4302cf', namespace = 'ai-tenants' def _delete(kind, name, namespace=None, *, timeout="60s"): args = ["delete", kind, name, "--ignore-not-found", f"--timeout={timeout}"] if namespace: args.extend(["-n", namespace]) process_timeout = OC_TIMEOUT + 30 if timeout.endswith("s") and timeout[:-1].isdigit(): process_timeout = max(process_timeout, int(timeout[:-1]) + 30) result = _oc_run(args, timeout=process_timeout) if result.returncode != 0: > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc delete aitenant e2e-ait-ed4302cf --ignore-not-found --timeout=180s -n ai-tenants` failed: error: timed out waiting for the condition on aitenants/e2e-ait-ed4302cf test/e2e/tests/test_aitenant_lifecycle.py:82: RuntimeError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:353: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7fb719abaf40> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase="Active", ) wait_for_aitenant_cleanup_resources(case) user_objects = apply_unrelated_tenant_objects(case) delete_best_effort(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout="180s") wait_for_not_found(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) wait_for_not_found("maasauthpolicy", case["policy_name"], case["tenant_ns"], timeout=180) wait_for_not_found("maassubscription", case["subscription_name"], case["tenant_ns"], timeout=180) wait_for_not_found("role", role_name, case["tenant_ns"], timeout=180) > namespace = wait_for_json( "namespace", case["tenant_ns"], predicate=lambda obj: LABEL_MANAGED_BY_AITENANT not in ((obj.get("metadata") or {}).get("labels") or {}), timeout=180, ) test/e2e/tests/test_multi_tenant_integration.py:121: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'namespace', name = 'ai-tenant-e2e-mt-534ea2a9', namespace = None def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: namespace/ai-tenant-e2e-mt-534ea2a9 in <cluster> did not satisfy condition. Last object: {'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': {'annotations': {'maas.opendatahub.io/aitenant-name': 'e2e-mt-534ea2a9', 'maas.opendatahub.io/aitenant-namespace': 'ai-tenants', 'maas.opendatahub.io/created-by-aitenant': 'true', 'openshift.io/sa.scc.mcs': 's0:c30,c25', 'openshift.io/sa.scc.supplemental-groups': '1000920000/10000', 'openshift.io/sa.scc.uid-range': '1000920000/10000', 'security.openshift.io/MinimallySufficientPodSecurityStandard': 'restricted'}, 'creationTimestamp': '2026-07-21T15:52:46Z', 'labels': {'ai-gateway.opendatahub.io/tenant': 'e2e-mt-534ea2a9', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'modelsasservice', 'kubernetes.io/metadata.name': 'ai-tenant-e2e-mt-534ea2a9', 'maas.opendatahub.io/gateway-access-e2e-mt-534ea2a9': 'true', 'maas.opendatahub.io/managed-by-aitenant': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-mt-534ea2a9', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-mt-534ea2a9', 'opendatahub.io/generated-namespace': 'true', 'pod-security.kubernetes.io/audit': 'restricted', 'pod-security.kubernetes.io/audit-version': 'latest', 'pod-security.kubernetes.io/warn': 'restricted', 'pod-security.kubernetes.io/warn-version': 'latest'}, 'name': 'ai-tenant-e2e-mt-534ea2a9', 'resourceVersion': '59588', 'uid': '4c63f158-6b4e-417c-945a-2f64d0d91ebb'}, 'spec': {'finalizers': ['kubernetes']}, 'status': {'phase': 'Active'}} test/e2e/tests/multitenancy_helpers.py:221: AssertionErrortenant_auth_setup = {'policy': 'e2e-auth-iso-b19562', 'subscription': 'e2e-auth-iso-b19562', 'tenant_a': {'base_url': 'https://e2e-shared-...me': 'auth-test-model-bbeb7d', 'model_namespace': 'ai-tenant-e2e-shared-b-bbeb7d', 'name': 'e2e-shared-b-bbeb7d', ...}} @pytest.fixture def tenant_api_keys(tenant_auth_setup): oc_token = _get_cluster_token() created = {} for key_name, tenant in (("a", tenant_auth_setup["tenant_a"]), ("b", tenant_auth_setup["tenant_b"])): response = create_api_key_at( tenant["base_url"], oc_token, f"e2e-auth-iso-{key_name}-{uuid.uuid4().hex[:6]}", subscription=tenant_auth_setup["subscription"], ) > assert response.status_code in (200, 201), ( f"create key for {tenant['name']} failed: {response_summary(response)}" ) E AssertionError: create key for e2e-shared-a-524b80 failed: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_auth_isolation.py:159: AssertionErrortenant_auth_setup = {'policy': 'e2e-auth-iso-811bb8', 'subscription': 'e2e-auth-iso-811bb8', 'tenant_a': {'base_url': 'https://e2e-shared-...me': 'auth-test-model-bbeb7d', 'model_namespace': 'ai-tenant-e2e-shared-b-bbeb7d', 'name': 'e2e-shared-b-bbeb7d', ...}} @pytest.fixture def tenant_api_keys(tenant_auth_setup): oc_token = _get_cluster_token() created = {} for key_name, tenant in (("a", tenant_auth_setup["tenant_a"]), ("b", tenant_auth_setup["tenant_b"])): response = create_api_key_at( tenant["base_url"], oc_token, f"e2e-auth-iso-{key_name}-{uuid.uuid4().hex[:6]}", subscription=tenant_auth_setup["subscription"], ) > assert response.status_code in (200, 201), ( f"create key for {tenant['name']} failed: {response_summary(response)}" ) E AssertionError: create key for e2e-shared-a-524b80 failed: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_auth_isolation.py:159: AssertionError