/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:321: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:378: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:454: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6f379f4130> def test_multiple_distinct_models_in_subscription(self): """ Test 8: Multiple distinct models should return exactly 2 entries (1 per unique ID). Uses pre-deployed models (both known to not have backend duplication issues): - DISTINCT_MODEL_REF (simulated-distinct) serving "test/e2e-distinct-model" - DISTINCT_MODEL_2_REF (simulated-distinct-2) serving "test/e2e-distinct-model-2" Creates a subscription with both models. The API should return exactly 2 entries (one for each distinct model ID), with no duplicates. This test validates that when backend models don't have duplication bugs, the API correctly returns one entry per distinct model ID. """ log.info("Test 8: Multiple distinct models should return 2 entries") sa_name = "e2e-models-distinct-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-distinct-models-subscription" auth_policy_name = "e2e-distinct-models-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both distinct models log.info(f"Creating auth policy with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both distinct models log.info(f"Creating subscription with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-distinct-models-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models log.info(f"Querying /v1/models with subscription: {subscription_name}") r = requests.get( f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 modelRefs ({DISTINCT_MODEL_REF}, {DISTINCT_MODEL_2_REF})") # Verify we got BOTH expected model IDs expected_ids = {DISTINCT_MODEL_ID, DISTINCT_MODEL_2_ID} > assert unique_ids == expected_ids, \ f"Expected to find both distinct models {expected_ids}, but got {unique_ids}" E AssertionError: Expected to find both distinct models {'test/e2e-distinct-model', 'test/e2e-distinct-model-2'}, but got {'test/e2e-distinct-model'} E assert {'test/e2e-distinct-model'} == {'test/e2e-di...inct-model-2'} E E Extra items in the right set: E 'test/e2e-distinct-model-2' E E Full diff: E { E 'test/e2e-distinct-model', E - 'test/e2e-distinct-model-2', E } test/e2e/tests/test_models_endpoint.py:1048: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6f3752f700> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403 with error type: permission_error """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_reconcile() # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_reconcile() # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = requests.get( f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, timeout=TIMEOUT, verify=TLS_VERIFY, ) # Should return 403 because subscription doesn't exist > assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for API key with deleted subscription, got 500: {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"001"} E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_models_endpoint.py:1529: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6f3752f2b0> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" _wait_reconcile() # Test key1 - should return models from sub1 only log.info(f"Testing API key 1 (bound to {sub1_name})") r1 = requests.get( f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {api_key1}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r1.status_code == 200, f"Expected 200 for key1, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} assert DISTINCT_MODEL_ID in model_ids1, f"Key1 should see {DISTINCT_MODEL_ID} from {sub1_name}" assert DISTINCT_MODEL_2_ID not in model_ids1, f"Key1 should NOT see {DISTINCT_MODEL_2_ID} from {sub2_name}" # Test key2 - should return models from sub2 only log.info(f"Testing API key 2 (bound to {sub2_name})") r2 = requests.get( f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {api_key2}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r2.status_code == 200, f"Expected 200 for key2, got {r2.status_code}: {r2.text}" models2 = r2.json().get("data") or [] model_ids2 = {m["id"] for m in models2} > assert DISTINCT_MODEL_2_ID in model_ids2, f"Key2 should see {DISTINCT_MODEL_2_ID} from {sub2_name}" E AssertionError: Key2 should see test/e2e-distinct-model-2 from e2e-multi-keys-sub2 E assert 'test/e2e-distinct-model-2' in set() test/e2e/tests/test_models_endpoint.py:1913: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6f379f4be0> def test_service_account_token_multiple_subs_no_header(self): """ Test: K8s token with access to multiple subscriptions returns all models (no header). Creates a service account with access to two subscriptions (via group and user). When querying without x-maas-subscription header, should return models from all accessible subscriptions. Expected: HTTP 200 with models from both subscriptions. """ sa_name = "e2e-sa-multi-subs-no-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-no-hdr-sub1" sub2_name = "e2e-sa-multi-no-hdr-sub2" auth1_name = "e2e-sa-multi-no-hdr-auth1" auth2_name = "e2e-sa-multi-no-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models # Sub1: Access via system:authenticated group log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF} (group: system:authenticated)") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) # Sub2: Access via specific user log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF} (user: {sa_user})") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token (no header) log.info("Querying /v1/models with K8s token (no header) - should return models from both subscriptions") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {sa_token}"}, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] model_ids = {m["id"] for m in models} # Should see models from BOTH subscriptions assert DISTINCT_MODEL_ID in model_ids, \ f"Should see {DISTINCT_MODEL_ID} from {sub1_name} (group access)" > assert DISTINCT_MODEL_2_ID in model_ids, \ f"Should see {DISTINCT_MODEL_2_ID} from {sub2_name} (user access)" E AssertionError: Should see test/e2e-distinct-model-2 from e2e-sa-multi-no-hdr-sub2 (user access) E assert 'test/e2e-distinct-model-2' in {'facebook/opt-125m', 'test/e2e-distinct-model'} test/e2e/tests/test_models_endpoint.py:1981: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6f379f4370> def test_service_account_token_multiple_subs_with_header(self): """ Test: K8s token with access to multiple subscriptions filters by header. Creates a service account with access to two subscriptions. When querying with x-maas-subscription header, should return models from only the specified subscription. Expected: HTTP 200 with models from only the specified subscription. """ sa_name = "e2e-sa-multi-subs-with-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-hdr-sub1" sub2_name = "e2e-sa-multi-hdr-sub2" auth1_name = "e2e-sa-multi-hdr-auth1" auth2_name = "e2e-sa-multi-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token and header specifying sub1 log.info(f"Querying /v1/models with K8s token and header: {sub1_name}") r1 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub1_name, }, ) assert r1.status_code == 200, f"Expected 200, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} # Should see only models from sub1 assert DISTINCT_MODEL_ID in model_ids1, f"Should see {DISTINCT_MODEL_ID} from {sub1_name}" assert DISTINCT_MODEL_2_ID not in model_ids1, f"Should NOT see {DISTINCT_MODEL_2_ID} from {sub2_name}" # Query with K8s token and header specifying sub2 log.info(f"Querying /v1/models with K8s token and header: {sub2_name}") r2 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub2_name, }, ) assert r2.status_code == 200, f"Expected 200, got {r2.status_code}: {r2.text}" models2 = r2.json().get("data") or [] model_ids2 = {m["id"] for m in models2} # Should see only models from sub2 > assert DISTINCT_MODEL_2_ID in model_ids2, f"Should see {DISTINCT_MODEL_2_ID} from {sub2_name}" E AssertionError: Should see test/e2e-distinct-model-2 from e2e-sa-multi-hdr-sub2 E assert 'test/e2e-distinct-model-2' in set() test/e2e/tests/test_models_endpoint.py:2066: AssertionError