/workspace/source/test/e2e/tests/test_subscription.py:1040: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7eff74d187c0> def test_multiple_distinct_models_in_subscription(self): """ Test 8: Multiple distinct models should return exactly 2 entries (1 per unique ID). Uses pre-deployed models (both known to not have backend duplication issues): - DISTINCT_MODEL_REF (simulated-distinct) serving "test/e2e-distinct-model" - DISTINCT_MODEL_2_REF (simulated-distinct-2) serving "test/e2e-distinct-model-2" Creates a subscription with both models. The API should return exactly 2 entries (one for each distinct model ID), with no duplicates. This test validates that when backend models don't have duplication bugs, the API correctly returns one entry per distinct model ID. """ log.info("Test 8: Multiple distinct models should return 2 entries") sa_name = "e2e-models-distinct-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-distinct-models-subscription" auth_policy_name = "e2e-distinct-models-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both distinct models log.info(f"Creating auth policy with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both distinct models log.info(f"Creating subscription with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-distinct-models-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models (use retry helper for gateway/Authorino propagation) log.info(f"Querying /v1/models with subscription: {subscription_name}") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 modelRefs ({DISTINCT_MODEL_REF}, {DISTINCT_MODEL_2_REF})") # Verify we got BOTH expected model IDs expected_ids = {DISTINCT_MODEL_ID, DISTINCT_MODEL_2_ID} > assert unique_ids == expected_ids, \ f"Expected to find both distinct models {expected_ids}, but got {unique_ids}" E AssertionError: Expected to find both distinct models {'test/e2e-distinct-model', 'test/e2e-distinct-model-2'}, but got {'test/e2e-distinct-model'} E assert {'test/e2e-distinct-model'} == {'test/e2e-di...inct-model-2'} E E Extra items in the right set: E 'test/e2e-distinct-model-2' E E Full diff: E { E 'test/e2e-distinct-model', E - 'test/e2e-distinct-model-2', E } test/e2e/tests/test_models_endpoint.py:1053: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7eff7528dca0> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" _wait_reconcile() # Test key1 - should return models from sub1 only log.info(f"Testing API key 1 (bound to {sub1_name})") r1 = requests.get( f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {api_key1}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r1.status_code == 200, f"Expected 200 for key1, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} assert DISTINCT_MODEL_ID in model_ids1, f"Key1 should see {DISTINCT_MODEL_ID} from {sub1_name}" assert DISTINCT_MODEL_2_ID not in model_ids1, f"Key1 should NOT see {DISTINCT_MODEL_2_ID} from {sub2_name}" # Test key2 - should return models from sub2 only log.info(f"Testing API key 2 (bound to {sub2_name})") r2 = requests.get( f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {api_key2}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r2.status_code == 200, f"Expected 200 for key2, got {r2.status_code}: {r2.text}" models2 = r2.json().get("data") or [] model_ids2 = {m["id"] for m in models2} > assert DISTINCT_MODEL_2_ID in model_ids2, f"Key2 should see {DISTINCT_MODEL_2_ID} from {sub2_name}" E AssertionError: Key2 should see test/e2e-distinct-model-2 from e2e-multi-keys-sub2 E assert 'test/e2e-distinct-model-2' in set() test/e2e/tests/test_models_endpoint.py:1918: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7eff752f27c0> def test_service_account_token_multiple_subs_no_header(self): """ Test: K8s token with access to multiple subscriptions returns all models (no header). Creates a service account with access to two subscriptions (via group and user). When querying without x-maas-subscription header, should return models from all accessible subscriptions. Expected: HTTP 200 with models from both subscriptions. """ sa_name = "e2e-sa-multi-subs-no-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-no-hdr-sub1" sub2_name = "e2e-sa-multi-no-hdr-sub2" auth1_name = "e2e-sa-multi-no-hdr-auth1" auth2_name = "e2e-sa-multi-no-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models # Sub1: Access via system:authenticated group log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF} (group: system:authenticated)") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) # Sub2: Access via specific user log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF} (user: {sa_user})") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token (no header) log.info("Querying /v1/models with K8s token (no header) - should return models from both subscriptions") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {sa_token}"}, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] model_ids = {m["id"] for m in models} # Should see models from BOTH subscriptions assert DISTINCT_MODEL_ID in model_ids, \ f"Should see {DISTINCT_MODEL_ID} from {sub1_name} (group access)" > assert DISTINCT_MODEL_2_ID in model_ids, \ f"Should see {DISTINCT_MODEL_2_ID} from {sub2_name} (user access)" E AssertionError: Should see test/e2e-distinct-model-2 from e2e-sa-multi-no-hdr-sub2 (user access) E assert 'test/e2e-distinct-model-2' in {'facebook/opt-125m', 'test/e2e-distinct-model'} test/e2e/tests/test_models_endpoint.py:1986: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7eff74d18af0> def test_service_account_token_multiple_subs_with_header(self): """ Test: K8s token with access to multiple subscriptions filters by header. Creates a service account with access to two subscriptions. When querying with x-maas-subscription header, should return models from only the specified subscription. Expected: HTTP 200 with models from only the specified subscription. """ sa_name = "e2e-sa-multi-subs-with-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-hdr-sub1" sub2_name = "e2e-sa-multi-hdr-sub2" auth1_name = "e2e-sa-multi-hdr-auth1" auth2_name = "e2e-sa-multi-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token and header specifying sub1 log.info(f"Querying /v1/models with K8s token and header: {sub1_name}") r1 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub1_name, }, ) assert r1.status_code == 200, f"Expected 200, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} # Should see only models from sub1 assert DISTINCT_MODEL_ID in model_ids1, f"Should see {DISTINCT_MODEL_ID} from {sub1_name}" assert DISTINCT_MODEL_2_ID not in model_ids1, f"Should NOT see {DISTINCT_MODEL_2_ID} from {sub2_name}" # Query with K8s token and header specifying sub2 log.info(f"Querying /v1/models with K8s token and header: {sub2_name}") r2 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub2_name, }, ) assert r2.status_code == 200, f"Expected 200, got {r2.status_code}: {r2.text}" models2 = r2.json().get("data") or [] model_ids2 = {m["id"] for m in models2} # Should see only models from sub2 > assert DISTINCT_MODEL_2_ID in model_ids2, f"Should see {DISTINCT_MODEL_2_ID} from {sub2_name}" E AssertionError: Should see test/e2e-distinct-model-2 from e2e-sa-multi-hdr-sub2 E assert 'test/e2e-distinct-model-2' in set() test/e2e/tests/test_models_endpoint.py:2071: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_external_oidc.py:426: OIDC_USERNAME_NO_ACCESS and OIDC_PASSWORD_NO_ACCESS not configured