/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7fcd205f0e50> def test_explicit_subscription_header(self): """ Test: K8s token with multiple subscriptions can list models by providing x-maas-subscription header. Expected: HTTP 200 with models from only the specified subscription. Note: Creates SA that has access to both simulator-subscription (via system:authenticated) and premium-simulator-subscription (by adding SA to its users list). Uses K8s token directly (not API key) since API keys ignore the header. """ sa_name = "e2e-models-explicit-header-sa" sa_ns = "default" maas_ns = _ns() sa_user = None try: # Create service account - will be in system:authenticated group # This gives access to simulator-subscription automatically sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Add SA to premium-simulator-subscription to give it access to a second subscription log.info(f"Adding {sa_user} to premium-simulator-subscription users") subprocess.run([ "oc", "patch", "maassubscription", PREMIUM_SIMULATOR_SUBSCRIPTION, "-n", maas_ns, "--type=merge", "-p", json.dumps({"spec": {"owner": {"users": [sa_user]}}}) ], check=True) _wait_reconcile() # Test: GET /v1/models WITH x-maas-subscription header using K8s token # Expected: Returns models from simulator-subscription only log.info("Testing: GET /v1/models with K8s token and explicit subscription header: simulator-subscription") url = f"{_maas_api_url()}/v1/models" r = _request_with_gateway_retry( requests.get, url, headers={ "Authorization": f"Bearer {sa_token}", # K8s token, not API key "x-maas-subscription": SIMULATOR_SUBSCRIPTION, }, ) assert r.status_code == 200, f"Expected 200 with explicit subscription header, got {r.status_code}: {r.text}" # Validate response structure data = r.json() assert data.get("object") == "list", f"Expected object='list', got {data.get('object')}" assert "data" in data, "Response missing 'data' field" models = data.get("data", []) if data.get("data") is not None else [] # Should have at least one model from simulator-subscription > assert len(models) > 0, f"Expected at least one model in response, got {len(models)}. Data was: {data.get('data')}" E AssertionError: Expected at least one model in response, got 0. Data was: [] E assert 0 > 0 E + where 0 = len([]) test/e2e/tests/test_models_endpoint.py:407: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fcd205f0610> def test_deduplication_same_model_multiple_refs(self): """ Test 6: Same modelRef listed twice should deduplicate to 1 entry. Creates a subscription with the SAME modelRef listed TWICE (different rate limits). The API deduplicates by (model ID, URL) and returns only 1 entry since both references point to the same backend service. The response includes subscription information showing which subscription(s) provide access to the model. """ log.info("Test 6: Same modelRef twice should deduplicate (INTENDED behavior)") sa_name = "e2e-models-dedup-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-dedup-subscription" auth_policy_name = "e2e-dedup-auth" api_key = None try: # Create SA with its own token sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy that grants access to the model log.info(f"Creating auth policy with access to {MODEL_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [{"name": MODEL_REF, "namespace": MODEL_NAMESPACE}], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with the SAME model ref TWICE (guaranteed duplicates) log.info(f"Creating subscription with {MODEL_REF} listed twice (to test deduplication)") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": MODEL_REF, # Same model ref again - guarantees duplicate "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 200, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-dedup-test-key", subscription=subscription_name) # Wait for reconciliation _wait_reconcile() # Query /v1/models with our custom subscription log.info(f"Querying /v1/models with subscription: {subscription_name}") r = _get_models_with_gateway_retry( headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] # Models should be a list assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") # Should all be the same model ID (we only referenced one modelRef) > assert len(unique_ids) == 1, \ f"Expected only 1 unique model ID (same modelRef listed twice), got {len(unique_ids)}: {unique_ids}" E AssertionError: Expected only 1 unique model ID (same modelRef listed twice), got 0: set() E assert 0 == 1 E + where 0 = len(set()) test/e2e/tests/test_models_endpoint.py:690: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fcd205f0730> def test_different_modelrefs_same_model_id(self): """ Test 7: Different modelRefs serving same model ID return separate entries. Uses two DIFFERENT MaaSModelRefs (each listed ONCE) that both serve the SAME model ID: - MODEL_REF (facebook-opt-125m-simulated) → serves "facebook/opt-125m" - PREMIUM_MODEL_REF (premium-simulated-simulated-premium) → serves "facebook/opt-125m" The API deduplicates by (model ID, URL). Since these are different backend services with different URLs, they return as 2 separate entries even though they serve the same model ID. Each entry shows the same model ID but different URL and subscription. """ log.info("Test 7: Different modelRefs same ID should deduplicate (INTENDED behavior)") sa_name = "e2e-models-diff-refs-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-diff-refs-subscription" auth_policy_name = "e2e-diff-refs-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both modelRefs log.info(f"Creating auth policy with {MODEL_REF} and {PREMIUM_MODEL_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both modelRefs (each listed ONCE) log.info(f"Creating subscription with {MODEL_REF} and {PREMIUM_MODEL_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": PREMIUM_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 200, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for auth and subscription reconciliation before creating API key. # Both modelRefs must be governed before /v1/models probes each backend. _wait_for_maas_auth_policy_phase( auth_policy_name, namespace=maas_ns, timeout=120, ) _wait_for_maas_subscription_phase( subscription_name, namespace=maas_ns, timeout=120, require_model_statuses=True, ) _wait_for_model_ready(MODEL_REF, namespace=MODEL_NAMESPACE) _wait_for_model_ready(PREMIUM_MODEL_REF, namespace=MODEL_NAMESPACE) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-diff-refs-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models. Gateway policy propagation and backend model probes can # lag behind CR readiness, so wait until both modelRefs appear. log.info(f"Querying /v1/models with subscription: {subscription_name}") request_headers = { "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, } deadline = time.time() + 120 r = None models = [] model_ids = [] while time.time() < deadline: r = _get_models_with_gateway_retry(headers=request_headers) if r.status_code == 200: models = r.json().get("data") or [] assert isinstance(models, list), "Models should be a list" model_ids = [m["id"] for m in models] urls = [m.get("url") for m in models if m.get("id") == MODEL_NAME and m.get("url")] if len(models) == 2 and len(set(urls)) == 2: break log.info( "Models response not fully propagated yet; " f"got {len(models)} entries: {model_ids}" ) else: log.info(f"Waiting for /v1/models HTTP 200; got {r.status_code}: {r.text[:200]}") time.sleep(5) assert r is not None, "Expected /v1/models response, got none" assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" assert isinstance(models, list), "Models should be a list" # Get model IDs from response unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(" Subscription had: 2 different modelRefs both serving 'facebook/opt-125m'") # Both modelRefs serve the same model ID > assert len(unique_ids) == 1, \ f"Expected only 1 unique model ID (both modelRefs serve {MODEL_NAME}), got {len(unique_ids)}: {unique_ids}" E AssertionError: Expected only 1 unique model ID (both modelRefs serve facebook/opt-125m), got 0: set() E assert 0 == 1 E + where 0 = len(set()) test/e2e/tests/test_models_endpoint.py:876: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fcd204b03a0> def test_multiple_distinct_models_in_subscription(self): """ Test 8: Multiple distinct models should return exactly 2 entries (1 per unique ID). Uses pre-deployed models (both known to not have backend duplication issues): - DISTINCT_MODEL_REF (simulated-distinct) serving "test/e2e-distinct-model" - DISTINCT_MODEL_2_REF (simulated-distinct-2) serving "test/e2e-distinct-model-2" Creates a subscription with both models. The API should return exactly 2 entries (one for each distinct model ID), with no duplicates. This test validates that when backend models don't have duplication bugs, the API correctly returns one entry per distinct model ID. """ log.info("Test 8: Multiple distinct models should return 2 entries") sa_name = "e2e-models-distinct-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-distinct-models-subscription" auth_policy_name = "e2e-distinct-models-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both distinct models log.info(f"Creating auth policy with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both distinct models log.info(f"Creating subscription with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Wait for models to become Ready after governance pairing is created log.info("Waiting for models to reconcile and become Ready...") _wait_for_model_ready(DISTINCT_MODEL_REF, namespace=MODEL_NAMESPACE) _wait_for_model_ready(DISTINCT_MODEL_2_REF, namespace=MODEL_NAMESPACE) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-distinct-models-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models (use retry helper for gateway/Authorino propagation) log.info(f"Querying /v1/models with subscription: {subscription_name}") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 modelRefs ({DISTINCT_MODEL_REF}, {DISTINCT_MODEL_2_REF})") # Verify we got BOTH expected model IDs expected_ids = {DISTINCT_MODEL_ID, DISTINCT_MODEL_2_ID} > assert unique_ids == expected_ids, \ f"Expected to find both distinct models {expected_ids}, but got {unique_ids}" E AssertionError: Expected to find both distinct models {'test/e2e-distinct-model', 'test/e2e-distinct-model-2'}, but got set() E assert set() == {'test/e2e-di...inct-model-2'} E E Extra items in the right set: E 'test/e2e-distinct-model' E 'test/e2e-distinct-model-2' E E Full diff: E + set()... E E ...Full output truncated (4 lines hidden), use '-vv' to show test/e2e/tests/test_models_endpoint.py:1043: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fcd1fe1c190> def test_user_token_returns_all_models(self): """ Test: User token automatically returns models from all subscriptions. Creates a user with access to TWO subscriptions containing different models. Queries without X-MaaS-Subscription header and validates: - Returns models from ALL accessible subscriptions - Each model includes subscriptions array showing which subscription(s) provide access - Models appearing in multiple subscriptions have aggregated subscription list """ log.info("Test: User token returns models from all subscriptions") sa_name = "e2e-return-all-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-return-all-sub1" sub2_name = "e2e-return-all-sub2" auth1_name = "e2e-return-all-auth1" auth2_name = "e2e-return-all-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create subscription 1 with DISTINCT_MODEL_REF log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) # Create subscription 2 with DISTINCT_MODEL_2_REF log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Wait for models to become Ready after governance pairing is created log.info("Waiting for models to reconcile and become Ready...") _wait_for_model_ready(DISTINCT_MODEL_REF, namespace=MODEL_NAMESPACE) _wait_for_model_ready(DISTINCT_MODEL_2_REF, namespace=MODEL_NAMESPACE) # Query with user token (no X-MaaS-Subscription header) log.info("Querying /v1/models with user token (no header)") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] # Should get models from BOTH subscriptions model_ids = [m["id"] for m in models] log.info(f"Got {len(models)} models: {model_ids}") # Validate we got models from both subscriptions # (At minimum we should see the 2 distinct models) > assert len(models) >= 2, \ f"Expected at least 2 models (from 2 subscriptions), got {len(models)}" E AssertionError: Expected at least 2 models (from 2 subscriptions), got 0 E assert 0 >= 2 E + where 0 = len([]) test/e2e/tests/test_models_endpoint.py:1137: AssertionError