/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_models_endpoint.TestModelsEndpoint object at 0x7ffab3e52610> def test_user_token_returns_all_models(self): """ Test: User token automatically returns models from all subscriptions. Creates a user with access to TWO subscriptions containing different models. Queries without X-MaaS-Subscription header and validates: - Returns models from ALL accessible subscriptions - Each model includes subscriptions array showing which subscription(s) provide access - Models appearing in multiple subscriptions have aggregated subscription list """ log.info("Test: User token returns models from all subscriptions") sa_name = "e2e-return-all-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-return-all-sub1" sub2_name = "e2e-return-all-sub2" auth1_name = "e2e-return-all-auth1" auth2_name = "e2e-return-all-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create subscription 1 with DISTINCT_MODEL_REF log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) # Create subscription 2 with DISTINCT_MODEL_2_REF log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_subscription_discovery_ready(sub1_name) _wait_for_subscription_discovery_ready(sub2_name) # Wait for models to become Ready after governance pairing is created log.info("Waiting for models to reconcile and become Ready...") _wait_for_model_ready(DISTINCT_MODEL_REF, namespace=MODEL_NAMESPACE) _wait_for_model_ready(DISTINCT_MODEL_2_REF, namespace=MODEL_NAMESPACE) # Query with user token (no X-MaaS-Subscription header) log.info("Querying /v1/models with user token (no header)") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", }, ) assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" data = r.json() models = data.get("data") or [] # Should get models from BOTH subscriptions model_ids = [m["id"] for m in models] log.info(f"Got {len(models)} models: {model_ids}") # Validate we got models from both subscriptions # (At minimum we should see the 2 distinct models) > assert len(models) >= 2, \ f"Expected at least 2 models (from 2 subscriptions), got {len(models)}" E AssertionError: Expected at least 2 models (from 2 subscriptions), got 1 E assert 1 >= 2 E + where 1 = len([{'id': 'publishers/e2e-models-e2e-worker-w3-4fc49b/models/e2e/distinct-w3-4fc49b', 'created': 1793627826, 'object': 'model', 'owned_by': 'e2e-models-e2e-worker-w3-4fc49b/distinct-w3-4fc49b', ...}]) test/maas-e2e/test/e2e/tests/test_models_endpoint.py:1306: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f635cb31d90> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403. The gateway may reject the revoked key before the request reaches maas-api, in which case the response body is empty. """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_subscription_discovery_ready(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_for_cr_absent("maassubscription", subscription_name, namespace=ns) # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist > assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for API key with deleted subscription, got 500: Internal Server Error. E E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/maas-e2e/test/e2e/tests/test_models_endpoint.py:1643: AssertionErrorcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w5-1d18fb' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/maas-e2e/test/e2e/tests/test_models_endpoint.py:381: Failedcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w5-1d18fb' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/maas-e2e/test/e2e/tests/test_models_endpoint.py:381: Failedcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w5-1d18fb' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/maas-e2e/test/e2e/tests/test_models_endpoint.py:381: Failedcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w5-1d18fb' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/maas-e2e/test/e2e/tests/test_models_endpoint.py:381: Failedcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w5-1d18fb' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/maas-e2e/test/e2e/tests/test_models_endpoint.py:381: Failed