self = <test_models_endpoint.TestModelsEndpoint object at 0x7f12eb79abe0> def test_single_subscription_auto_select(self): """ Test: User with exactly one accessible subscription can list models without providing x-maas-subscription header (auto-selection). Expected: HTTP 200 with models from that subscription. Note: Temporarily deletes simulator-subscription to ensure test user has exactly ONE subscription (not two, which would require a header). """ sa_name = "e2e-models-single-sub-sa" sa_ns = "default" maas_ns = _ns() auth_policy_name = "e2e-single-sub-auth" subscription_name = "e2e-single-sub-subscription" # Snapshot existing subscription to restore later original_sim = _snapshot_cr("maassubscription", SIMULATOR_SUBSCRIPTION) api_key = None try: # Create service account sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Delete simulator-subscription so user has exactly ONE subscription # (otherwise they'd have 2: ours + simulator-subscription via system:authenticated) _delete_cr("maassubscription", SIMULATOR_SUBSCRIPTION) # Create auth policy and subscription for test user using DISTINCT_MODEL_REF # (avoids conflicts with existing simulator-access auth policy) log.info(f"Creating auth policy and subscription for {sa_user} with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth_policy_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, DISTINCT_MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Create API key for inference api_key = _create_api_key(sa_token, name=f"{sa_name}-key") # Wait for Authorino to sync auth policies (can take 30+ seconds) log.info("Waiting 30s for Authorino to sync auth policies...") time.sleep(30) # DEBUG: Test model endpoint directly first log.info("DEBUG: Testing direct model endpoint access...") model_endpoint = f"https://{os.environ['GATEWAY_HOST']}/llm/{DISTINCT_MODEL_REF}/v1/models" debug_r = requests.get( model_endpoint, headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, timeout=TIMEOUT, verify=TLS_VERIFY, ) log.info(f"DEBUG: Direct model endpoint returned {debug_r.status_code}") if debug_r.status_code == 200: log.info(f"DEBUG: Direct model endpoint data: {debug_r.json()}") else: log.info(f"DEBUG: Direct model endpoint error: {debug_r.text}") # Poll /v1/models until it returns models or timeout log.info("Testing: GET /v1/models with single subscription (no header, auto-select)") url = f"{_maas_api_url()}/v1/models" timeout_seconds = 60 poll_interval = 2 deadline = time.time() + timeout_seconds r = None while time.time() < deadline: r = requests.get( url, headers={"Authorization": f"Bearer {api_key}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) if r.status_code == 200: models = (r.json().get("data") or []) if len(models) > 0: log.info(f"✅ Models available after {60 - int(deadline - time.time())}s") break log.info(f"Got 200 but no models yet, retrying... ({int(deadline - time.time())}s remaining)") else: log.info(f"Got {r.status_code}, retrying... ({int(deadline - time.time())}s remaining)") time.sleep(poll_interval) assert r is not None and r.status_code == 200, f"Expected 200 for single subscription auto-select, got {r.status_code if r else 'timeout'}: {r.text if r else 'no response'}" # Validate response structure data = r.json() assert data.get("object") == "list", f"Expected object='list', got {data.get('object')}" assert "data" in data, "Response missing 'data' field" # Handle API bug: data may be null instead of [] models = data.get("data") or [] # Should have at least one model (facebook-opt-125m-simulated from simulator-subscription) > assert len(models) > 0, f"Expected at least one model in response, got {len(models)}. Data was: {data.get('data')}" E AssertionError: Expected at least one model in response, got 0. Data was: [] E assert 0 > 0 E + where 0 = len([]) test/e2e/tests/test_models_endpoint.py:344: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f12ebd39ee0> def test_multiple_distinct_models_in_subscription(self): """ Test 8: Multiple distinct models should return exactly 2 entries (1 per unique ID). Uses pre-deployed models (both known to not have backend duplication issues): - DISTINCT_MODEL_REF (simulated-distinct) serving "test/e2e-distinct-model" - DISTINCT_MODEL_2_REF (simulated-distinct-2) serving "test/e2e-distinct-model-2" Creates a subscription with both models. The API should return exactly 2 entries (one for each distinct model ID), with no duplicates. This test validates that when backend models don't have duplication bugs, the API correctly returns one entry per distinct model ID. """ log.info("Test 8: Multiple distinct models should return 2 entries") sa_name = "e2e-models-distinct-sa" sa_ns = "default" maas_ns = _ns() subscription_name = "e2e-distinct-models-subscription" auth_policy_name = "e2e-distinct-models-auth" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create auth policy with both distinct models log.info(f"Creating auth policy with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") auth_policy_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": { "name": auth_policy_name, "namespace": maas_ns, }, "spec": { "modelRefs": [ {"name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE}, {"name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE}, ], "subjects": { "users": [sa_user], "groups": [], }, }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(auth_policy_cr), text=True, check=True, ) # Create subscription with both distinct models log.info(f"Creating subscription with {DISTINCT_MODEL_REF} and {DISTINCT_MODEL_2_REF}") subscription_cr = { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": { "name": subscription_name, "namespace": maas_ns, }, "spec": { "owner": { "users": [sa_user], "groups": [], }, "modelRefs": [ { "name": DISTINCT_MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, { "name": DISTINCT_MODEL_2_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}], }, ], }, } subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps(subscription_cr), text=True, check=True, ) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) # Create API key bound to our test subscription api_key = _create_api_key(sa_token, name="e2e-distinct-models-test-key", subscription=subscription_name) _wait_reconcile() # Query /v1/models log.info(f"Querying /v1/models with subscription: {subscription_name}") r = requests.get( f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": subscription_name, }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] assert isinstance(models, list), "Models should be a list" # Get model IDs from response model_ids = [m["id"] for m in models] unique_ids = set(model_ids) log.info(f"#x1F4CA API Response: {len(models)} total model(s), {len(unique_ids)} unique ID(s)") log.info(f" Model IDs: {model_ids}") log.info(f" Unique IDs: {unique_ids}") log.info(f" Subscription had: 2 modelRefs ({DISTINCT_MODEL_REF}, {DISTINCT_MODEL_2_REF})") # Verify we got BOTH expected model IDs expected_ids = {DISTINCT_MODEL_ID, DISTINCT_MODEL_2_ID} > assert unique_ids == expected_ids, \ f"Expected to find both distinct models {expected_ids}, but got {unique_ids}" E AssertionError: Expected to find both distinct models {'test/e2e-distinct-model-2', 'test/e2e-distinct-model'}, but got {'test/e2e-distinct-model-2'} E assert {'test/e2e-distinct-model-2'} == {'test/e2e-di...inct-model-2'} E E Extra items in the right set: E 'test/e2e-distinct-model' E E Full diff: E { E - 'test/e2e-distinct-model', E 'test/e2e-distinct-model-2', E } test/e2e/tests/test_models_endpoint.py:1046: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f12eb860a30> def test_api_key_ignores_subscription_header(self): """ Test: API key ignores x-maas-subscription header and uses bound subscription. Creates an API key bound to one subscription, then sends request with header pointing to a different subscription. The API key should ignore the header and return models from its bound subscription. Expected: HTTP 200 with models from the key's bound subscription (header ignored). """ sa_name = "e2e-api-key-ignores-header-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-ignore-header-sub1" sub2_name = "e2e-ignore-header-sub2" auth1_name = "e2e-ignore-header-auth1" auth2_name = "e2e-ignore-header-auth2" api_key = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user], priority=10) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user], priority=5) # Wait for both subscriptions to reconcile before creating API key _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create API key - will be bound to highest priority subscription (sub1) log.info(f"Creating API key (will bind to {sub1_name} - highest priority)") api_key = _create_api_key(sa_token, name=f"{sa_name}-key") _wait_reconcile() # Test: Send request with header pointing to sub2, but key is bound to sub1 log.info(f"Querying /v1/models with API key bound to {sub1_name} but header={sub2_name}") r = requests.get( f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", "x-maas-subscription": sub2_name, # Try to override with header }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] # Verify we got models from sub1 (not sub2 - header ignored) > assert len(models) > 0, "Expected at least one model" E AssertionError: Expected at least one model E assert 0 > 0 E + where 0 = len([]) test/e2e/tests/test_models_endpoint.py:1796: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f12eb860af0> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_maas_subscription_phase(sub1_name, namespace=maas_ns) _wait_for_maas_subscription_phase(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" _wait_reconcile() # Test key1 - should return models from sub1 only log.info(f"Testing API key 1 (bound to {sub1_name})") r1 = requests.get( f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {api_key1}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert r1.status_code == 200, f"Expected 200 for key1, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} > assert DISTINCT_MODEL_ID in model_ids1, f"Key1 should see {DISTINCT_MODEL_ID} from {sub1_name}" E AssertionError: Key1 should see test/e2e-distinct-model from e2e-multi-keys-sub1 E assert 'test/e2e-distinct-model' in set() test/e2e/tests/test_models_endpoint.py:1896: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f12ebd39610> def test_service_account_token_multiple_subs_no_header(self): """ Test: K8s token with access to multiple subscriptions returns all models (no header). Creates a service account with access to two subscriptions (via group and user). When querying without x-maas-subscription header, should return models from all accessible subscriptions. Expected: HTTP 200 with models from both subscriptions. """ sa_name = "e2e-sa-multi-subs-no-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-no-hdr-sub1" sub2_name = "e2e-sa-multi-no-hdr-sub2" auth1_name = "e2e-sa-multi-no-hdr-auth1" auth2_name = "e2e-sa-multi-no-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models # Sub1: Access via system:authenticated group log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF} (group: system:authenticated)") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, groups=["system:authenticated"]) # Sub2: Access via specific user log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF} (user: {sa_user})") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token (no header) log.info("Querying /v1/models with K8s token (no header) - should return models from both subscriptions") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={"Authorization": f"Bearer {sa_token}"}, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] model_ids = {m["id"] for m in models} # Should see models from BOTH subscriptions > assert DISTINCT_MODEL_ID in model_ids, \ f"Should see {DISTINCT_MODEL_ID} from {sub1_name} (group access)" E AssertionError: Should see test/e2e-distinct-model from e2e-sa-multi-no-hdr-sub1 (group access) E assert 'test/e2e-distinct-model' in {'facebook/opt-125m', 'test/e2e-distinct-model-2'} test/e2e/tests/test_models_endpoint.py:1977: AssertionError