/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7f14c61714f0> def test_multiple_distinct_models_in_subscription(self): """ Test 8: Multiple distinct models should return exactly 2 entries (1 per unique ID). Uses pre-deployed models (both known to not have backend duplication issues): - DISTINCT_MODEL_REF (simulated-distinct) serving "test/e2e-distinct-model" - DISTINCT_MODEL_2_REF (simulated-distinct-2) serving "test/e2e-distinct-model-2" Creates a subscription with both models. The API should return exactly 2 entries (one for each distinct model ID), with no duplicates. This test validates that when backend models don't have duplication bugs, the API correctly returns one entry per distinct model ID. """ log.info("Test 8: Multiple distinct models should return 2 entries") sa_name = "e2e-models-distinct-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-distinct-models-subscription" auth_policy_name = "e2e-distinct-models-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both distinct models log.info(f"Creating auth policy with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both distinct models log.info(f"Creating subscription with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-distinct-models-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models (use retry helper for gateway/Authorino propagation) log.info(f"Querying /v1/models with subscription: {subscription_name}") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 modelRefs ({DISTINCT_MODEL_REF}, {DISTINCT_MODEL_2_REF})") # Verify we got BOTH expected model IDs expected_ids = {DISTINCT_MODEL_ID, DISTINCT_MODEL_2_ID} > assert unique_ids == expected_ids, \ f"Expected to find both distinct models {expected_ids}, but got {unique_ids}" E AssertionError: Expected to find both distinct models {'test/e2e-distinct-model-2', 'test/e2e-distinct-model'}, but got {'test/e2e-distinct-model-2'} E assert {'test/e2e-distinct-model-2'} == {'test/e2e-di...inct-model-2'} E E Extra items in the right set: E 'test/e2e-distinct-model' E E Full diff: E { E - 'test/e2e-distinct-model', E 'test/e2e-distinct-model-2', E } test/e2e/tests/test_models_endpoint.py:1053: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f14c61477c0> def test_api_key_ignores_subscription_header(self): """ Test: API key ignores x-maas-subscription header and uses bound subscription. Creates an API key bound to one subscription, then sends request with header pointing to a different subscription. The API key should ignore the header and return models from its bound subscription. Expected: HTTP 200 with models from the key's bound subscription (header ignored). """ sa_name = "e2e-api-key-ignores-header-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-ignore-header-sub1" sub2_name = "e2e-ignore-header-sub2" auth1_name = "e2e-ignore-header-auth1" auth2_name = "e2e-ignore-header-auth2" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user], priority=10) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user], priority=5) # Wait for both subscriptions to reconcile before creating API key _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create API key - will be bound to highest priority subscription (sub1) log.info(f"Creating API key (will bind to {sub1_name} - highest priority)") api_key = _create_api_key(sa_token, name=f"{sa_name}-key") _wait_reconcile() # Test: Send request with header pointing to sub2, but key is bound to sub1 log.info(f"Querying /v1/models with API key bound to {sub1_name} but header={sub2_name}") r = requests.get( f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": sub2_name, # Try to override with header }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] # Verify we got models from sub1 (not sub2 - header ignored) > assert len(models) > 0, "Expected at least one model" E AssertionError: Expected at least one model E assert 0 > 0 E + where 0 = len([]) test/e2e/tests/test_models_endpoint.py:1803: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f14c6147b50> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" _wait_reconcile() # Test key1 - should return models from sub1 only log.info(f"Testing API key 1 (bound to {sub1_name})") r1 = requests.get( f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {api_key1}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r1.status_code == 200, f"Expected 200 for key1, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} > assert DISTINCT_MODEL_ID in model_ids1, f"Key1 should see {DISTINCT_MODEL_ID} from {sub1_name}" E AssertionError: Key1 should see test/e2e-distinct-model from e2e-multi-keys-sub1 E assert 'test/e2e-distinct-model' in set() test/e2e/tests/test_models_endpoint.py:1903: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f14c6171340> def test_service_account_token_multiple_subs_no_header(self): """ Test: K8s token with access to multiple subscriptions returns all models (no header). Creates a service account with access to two subscriptions (via group and user). When querying without x-maas-subscription header, should return models from all accessible subscriptions. Expected: HTTP 200 with models from both subscriptions. """ sa_name = "e2e-sa-multi-subs-no-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-no-hdr-sub1" sub2_name = "e2e-sa-multi-no-hdr-sub2" auth1_name = "e2e-sa-multi-no-hdr-auth1" auth2_name = "e2e-sa-multi-no-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models # Sub1: Access via system:authenticated group log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF} (group: system:authenticated)") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) # Sub2: Access via specific user log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF} (user: {sa_user})") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token (no header) log.info("Querying /v1/models with K8s token (no header) - should return models from both subscriptions") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {sa_token}"}, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] model_ids = {m["id"] for m in models} # Should see models from BOTH subscriptions > assert DISTINCT_MODEL_ID in model_ids, \ f"Should see {DISTINCT_MODEL_ID} from {sub1_name} (group access)" E AssertionError: Should see test/e2e-distinct-model from e2e-sa-multi-no-hdr-sub1 (group access) E assert 'test/e2e-distinct-model' in {'facebook/opt-125m', 'test/e2e-distinct-model-2'} test/e2e/tests/test_models_endpoint.py:1984: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f14c5bc4f10> def test_service_account_token_multiple_subs_with_header(self): """ Test: K8s token with access to multiple subscriptions filters by header. Creates a service account with access to two subscriptions. When querying with x-maas-subscription header, should return models from only the specified subscription. Expected: HTTP 200 with models from only the specified subscription. """ sa_name = "e2e-sa-multi-subs-with-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-hdr-sub1" sub2_name = "e2e-sa-multi-hdr-sub2" auth1_name = "e2e-sa-multi-hdr-auth1" auth2_name = "e2e-sa-multi-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token and header specifying sub1 log.info(f"Querying /v1/models with K8s token and header: {sub1_name}") r1 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub1_name, }, ) assert r1.status_code == 200, f"Expected 200, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} # Should see only models from sub1 > assert DISTINCT_MODEL_ID in model_ids1, f"Should see {DISTINCT_MODEL_ID} from {sub1_name}" E AssertionError: Should see test/e2e-distinct-model from e2e-sa-multi-hdr-sub1 E assert 'test/e2e-distinct-model' in set() test/e2e/tests/test_models_endpoint.py:2052: AssertionError