/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7fa8ff139310> def test_different_modelrefs_same_model_id(self): """ Test 7: Different modelRefs serving same model ID return separate entries. Uses two DIFFERENT MaaSModelRefs (each listed ONCE) that both serve the SAME model ID: - MODEL_REF (facebook-opt-125m-simulated) → serves "facebook/opt-125m" - PREMIUM_MODEL_REF (premium-simulated-simulated-premium) → serves "facebook/opt-125m" The API deduplicates by (model ID, URL). Since these are different backend services with different URLs, they return as 2 separate entries even though they serve the same model ID. Each entry shows the same model ID but different URL and subscription. """ log.info("Test 7: Different modelRefs same ID should deduplicate (INTENDED behavior)") sa_name = "e2e-models-diff-refs-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-diff-refs-subscription" auth_policy_name = "e2e-diff-refs-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both modelRefs log.info(f"Creating auth policy with {MODEL_REF} and {PREMIUM_MODEL_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both modelRefs (each listed ONCE) log.info(f"Creating subscription with {MODEL_REF} and {PREMIUM_MODEL_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 200, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for auth and subscription reconciliation before creating API key. # Both modelRefs must be governed before /v1/models probes each backend. _wait_for_maas_auth_policy_phase( auth_policy_name, namespace=maas_ns, timeout=120, ) _wait_for_maas_subscription_phase( subscription_name, namespace=maas_ns, timeout=120, require_model_statuses=True, ) _wait_for_model_ready(MODEL_REF, namespace=MODEL_NAMESPACE) _wait_for_model_ready(PREMIUM_MODEL_REF, namespace=MODEL_NAMESPACE) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-diff-refs-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models. Gateway policy propagation and backend model probes can # lag behind CR readiness, so wait until both modelRefs appear. log.info(f"Querying /v1/models with subscription: {subscription_name}") request_headers = { "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, } deadline = time.time() + 120 r = None models = [] model_ids = [] while time.time() < deadline: r = _get_models_with_gateway_retry(headers=request_headers) if r.status_code == 200: models = r.json().get("data") or [] assert isinstance(models, list), "Models should be a list" model_ids = [m["id"] for m in models] urls = [m.get("url") for m in models if m.get("id") == MODEL_NAME and m.get("url")] if len(models) == 2 and len(set(urls)) == 2: break log.info( "Models response not fully propagated yet; " f"got {len(models)} entries: {model_ids}" ) else: log.info(f"Waiting for /v1/models HTTP 200; got {r.status_code}: {r.text[:200]}") time.sleep(5) assert r is not None, "Expected /v1/models response, got none" assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" assert isinstance(models, list), "Models should be a list" # Get model IDs from response unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(" Subscription had: 2 different modelRefs both serving 'facebook/opt-125m'") # Both modelRefs serve the same model ID assert len(unique_ids) == 1, \ f"Expected only 1 unique model ID (both modelRefs serve {MODEL_NAME}), got {len(unique_ids)}: {unique_ids}" # Verify it's the expected model ID expected_id = MODEL_NAME > assert expected_id in unique_ids, \ f"Expected to find '{expected_id}', but got {unique_ids}" E AssertionError: Expected to find 'facebook/opt-125m', but got {'publishers/llm/models/facebook/opt-125m'} E assert 'facebook/opt-125m' in {'publishers/llm/models/facebook/opt-125m'} test/e2e/tests/test_models_endpoint.py:881: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fa8ff12ba60> def test_multiple_distinct_models_in_subscription(self): """ Test 8: Multiple distinct models should return exactly 2 entries (1 per unique ID). Uses pre-deployed models (both known to not have backend duplication issues): - DISTINCT_MODEL_REF (simulated-distinct) serving "test/e2e-distinct-model" - DISTINCT_MODEL_2_REF (simulated-distinct-2) serving "test/e2e-distinct-model-2" Creates a subscription with both models. The API should return exactly 2 entries (one for each distinct model ID), with no duplicates. This test validates that when backend models don't have duplication bugs, the API correctly returns one entry per distinct model ID. """ log.info("Test 8: Multiple distinct models should return 2 entries") sa_name = "e2e-models-distinct-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-distinct-models-subscription" auth_policy_name = "e2e-distinct-models-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both distinct models log.info(f"Creating auth policy with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both distinct models log.info(f"Creating subscription with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Wait for models to become Ready after governance pairing is created log.info("Waiting for models to reconcile and become Ready...") _wait_for_model_ready(DISTINCT_MODEL_REF, namespace=MODEL_NAMESPACE) _wait_for_model_ready(DISTINCT_MODEL_2_REF, namespace=MODEL_NAMESPACE) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-distinct-models-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models (use retry helper for gateway/Authorino propagation) log.info(f"Querying /v1/models with subscription: {subscription_name}") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 modelRefs ({DISTINCT_MODEL_REF}, {DISTINCT_MODEL_2_REF})") # Verify we got BOTH expected model IDs expected_ids = {DISTINCT_MODEL_ID, DISTINCT_MODEL_2_ID} > assert unique_ids == expected_ids, \ f"Expected to find both distinct models {expected_ids}, but got {unique_ids}" E AssertionError: Expected to find both distinct models {'test/e2e-distinct-model-2', 'test/e2e-distinct-model'}, but got {'publishers/llm/models/test/e2e-distinct-model', 'publishers/llm/models/test/e2e-distinct-model-2'} E assert {'publishers/...inct-model-2'} == {'test/e2e-di...inct-model-2'} E E Extra items in the left set: E 'publishers/llm/models/test/e2e-distinct-model' E 'publishers/llm/models/test/e2e-distinct-model-2' E Extra items in the right set: E 'test/e2e-distinct-model-2' E 'test/e2e-distinct-model'... E E ...Full output truncated (8 lines hidden), use '-vv' to show test/e2e/tests/test_models_endpoint.py:1043: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fa8ff6e8d30> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" _wait_reconcile() # Test key1 - should return models from sub1 only log.info(f"Testing API key 1 (bound to {sub1_name})") r1 = _get_models_with_gateway_retry( headers={"Authorization": f"Bearer {api_key1}"}, ) assert r1.status_code == 200, f"Expected 200 for key1, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} > assert DISTINCT_MODEL_ID in model_ids1, f"Key1 should see {DISTINCT_MODEL_ID} from {sub1_name}" E AssertionError: Key1 should see test/e2e-distinct-model from e2e-multi-keys-sub1 E assert 'test/e2e-distinct-model' in {'publishers/llm/models/test/e2e-distinct-model'} test/e2e/tests/test_models_endpoint.py:1879: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fa8ff7b4880> def test_service_account_token_multiple_subs_no_header(self): """ Test: K8s token with access to multiple subscriptions returns all models (no header). Creates a service account with access to two subscriptions (via group and user). When querying without x-maas-subscription header, should return models from all accessible subscriptions. Expected: HTTP 200 with models from both subscriptions. """ sa_name = "e2e-sa-multi-subs-no-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-no-hdr-sub1" sub2_name = "e2e-sa-multi-no-hdr-sub2" auth1_name = "e2e-sa-multi-no-hdr-auth1" auth2_name = "e2e-sa-multi-no-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models # Sub1: Access via system:authenticated group log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF} (group: system:authenticated)") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) # Sub2: Access via specific user log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF} (user: {sa_user})") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token (no header) log.info("Querying /v1/models with K8s token (no header) - should return models from both subscriptions") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {sa_token}"}, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] model_ids = {m["id"] for m in models} # Should see models from BOTH subscriptions > assert DISTINCT_MODEL_ID in model_ids, \ f"Should see {DISTINCT_MODEL_ID} from {sub1_name} (group access)" E AssertionError: Should see test/e2e-distinct-model from e2e-sa-multi-no-hdr-sub1 (group access) E assert 'test/e2e-distinct-model' in {'publishers/llm/models/facebook/opt-125m', 'publishers/llm/models/test/e2e-distinct-model', 'publishers/llm/models/test/e2e-distinct-model-2'} test/e2e/tests/test_models_endpoint.py:1957: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fa8ff7b4730> def test_service_account_token_multiple_subs_with_header(self): """ Test: K8s token with access to multiple subscriptions filters by header. Creates a service account with access to two subscriptions. When querying with x-maas-subscription header, should return models from only the specified subscription. Expected: HTTP 200 with models from only the specified subscription. """ sa_name = "e2e-sa-multi-subs-with-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-hdr-sub1" sub2_name = "e2e-sa-multi-hdr-sub2" auth1_name = "e2e-sa-multi-hdr-auth1" auth2_name = "e2e-sa-multi-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token and header specifying sub1 log.info(f"Querying /v1/models with K8s token and header: {sub1_name}") r1 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub1_name, }, ) assert r1.status_code == 200, f"Expected 200, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} # Should see only models from sub1 > assert DISTINCT_MODEL_ID in model_ids1, f"Should see {DISTINCT_MODEL_ID} from {sub1_name}" E AssertionError: Should see test/e2e-distinct-model from e2e-sa-multi-hdr-sub1 E assert 'test/e2e-distinct-model' in {'publishers/llm/models/test/e2e-distinct-model'} test/e2e/tests/test_models_endpoint.py:2025: AssertionError