/workspace/source/test/e2e/tests/test_api_keys.py:930: CronJob maas-api-key-cleanup not found in opendatahub: Error from server (NotFound): cronjobs.batch "maas-api-key-cleanup" not found/workspace/source/test/e2e/tests/test_api_keys.py:977: NetworkPolicy maas-api-cleanup-restrict not found in opendatahub: Error from server (NotFound): networkpolicies.networking.k8s.io "maas-api-cleanup-restrict" not found/workspace/source/test/e2e/tests/test_api_keys.py:1096: Cannot find maas-api pod in opendatahub: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}self = <test_api_keys.TestAPIKeySubscriptionFilter object at 0x7f6d40bb0a60> api_keys_base_url = 'https://maas.apps.7d1d1265-0d1b-4223-85d8-71b0313a0e1a.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Ikw3Z3FvRF9RSld6ODI4LUE4eGpodHZTay1KUmt6QWtIdUhuQ0hpY3NsWGsifQ.e...OI88rHPf9Oxp_5kFLPSK6oSbiWbHm8VCgmfrUp5aYkuQu8Zf5iQxNTnS-elPmwO4SGrzEeTf1LhEYzEEQ', 'Content-Type': 'application/json'} def test_search_without_subscription_returns_all(self, api_keys_base_url: str, headers: dict): """Search without subscription filter returns keys across all subscriptions.""" key_ids = [] try: # Create keys with explicit subscription binding for i in range(2): r = requests.post( api_keys_base_url, headers=headers, json={"name": f"e2e-nofilter-{i}", "subscription": SIMULATOR_SUBSCRIPTION}, timeout=TIMEOUT, verify=TLS_VERIFY, ) > assert r.status_code in (200, 201), f"Failed to create key: {r.text}" E AssertionError: Failed to create key: E assert 403 in (200, 201) E + where 403 = <Response [403]>.status_code test/e2e/tests/test_api_keys.py:1591: AssertionError/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6d4072e220> def test_api_key_with_inaccessible_subscription_403(self): """ Test: API key bound to a subscription the user no longer has access to. This tests an edge case where an API key was minted when the user had access to a subscription, but later the user's group membership changed and they lost access. The key still has the subscription bound. Expected: HTTP 403 with error type: permission_error """ ns = _ns() auth_policy_name = "e2e-api-key-no-access-auth" subscription_name = "e2e-api-key-no-access-sub" sa_user = "e2e-api-key-user-sa" sa_other = "e2e-api-key-other-sa" try: # Create two service accounts oc_token_user = _create_sa_token(sa_user, namespace=ns) _ = _create_sa_token(sa_other, namespace=ns) user_principal = _sa_to_user(sa_user, namespace=ns) other_principal = _sa_to_user(sa_other, namespace=ns) # Create subscription accessible only to "other" user _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[user_principal, other_principal]) _create_test_subscription(subscription_name, MODEL_REF, users=[other_principal]) _wait_reconcile() # User tries to query with their token but specifying the other user's subscription # This simulates what would happen if an API key was bound to a subscription # the user doesn't have access to log.info("Querying /v1/models with user token and inaccessible subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {oc_token_user}", "X-MaaS-Subscription": subscription_name, }, ) # Should return 403 because user doesn't have access to the subscription > assert r.status_code == 403, \ f"Expected 403 for subscription without access, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for subscription without access, got 200: {"data":[{"id":"facebook/opt-125m","created":1783571941,"object":"model","owned_by":"llm/facebook-opt-125m-simulated","kind":"LLMInferenceService","url":"https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/facebook-opt-125m-simulated","ready":true,"modelDetails":{"description":"A simulated OPT-125M model for free-tier testing","displayName":"Facebook OPT 125M (Simulated)"},"subscriptions":[{"name":"simulator-subscription","displayName":"Simulator Subscription (Free)","description":"Free-tier subscription with 100 tokens/min rate limit"}]}],"object":"list"} E assert 200 == 403 E + where 200 = <Response [200]>.status_code test/e2e/tests/test_models_endpoint.py:1564: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6d4040b220> def test_invalid_subscription_header_403(self): """ Test: User with valid subscriptions but providing an invalid/non-existent subscription in the header gets 403. Expected: HTTP 403 with error type: permission_error and message: "requested subscription not found". """ ns = _ns() auth_policy_name = "e2e-models-invalid-sub-auth" subscription_name = "e2e-models-valid-sub" sa_name = "e2e-models-invalid-sub-sa" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources - user has valid subscription _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile() # Test: GET /v1/models WITH non-existent subscription header # Expected: 403 with "subscription not found" error invalid_sub = "nonexistent-subscription-xyz" log.info(f"Testing: GET /v1/models with invalid subscription header: {invalid_sub}") url = f"{_maas_api_url()}/v1/models" r = _request_with_gateway_retry( requests.get, url, headers={ "Authorization": f"Bearer {oc_token}", "x-maas-subscription": invalid_sub, }, ) > assert r.status_code == 403, f"Expected 403 for invalid subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for invalid subscription, got 200: {"data":[{"id":"facebook/opt-125m","created":1783571950,"object":"model","owned_by":"llm/facebook-opt-125m-simulated","kind":"LLMInferenceService","url":"https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/facebook-opt-125m-simulated","ready":true,"modelDetails":{"description":"A simulated OPT-125M model for free-tier testing","displayName":"Facebook OPT 125M (Simulated)"},"subscriptions":[{"name":"e2e-models-valid-sub"},{"name":"simulator-subscription","displayName":"Simulator Subscription (Free)","description":"Free-tier subscription with 100 tokens/min rate limit"}]}],"object":"list"} E assert 200 == 403 E + where 200 = <Response [200]>.status_code test/e2e/tests/test_models_endpoint.py:1620: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6d4040b280> def test_access_denied_to_subscription_403(self): """ Test: Subscription exists but user is not in its MaaSAuthPolicy owner list. User requests that subscription via header. Expected: HTTP 403 with error type: permission_error and message: "access denied to requested subscription". """ ns = _ns() auth_policy_name = "e2e-models-access-denied-auth" user_subscription = "e2e-models-user-sub" other_subscription = "e2e-models-other-sub" sa_user = "e2e-models-user-sa" sa_other = "e2e-models-other-sa" try: # Create two service accounts oc_token_user = _create_sa_token(sa_user, namespace=ns) _ = _create_sa_token(sa_other, namespace=ns) # SA creation only - token unused user_principal = _sa_to_user(sa_user, namespace=ns) other_principal = _sa_to_user(sa_other, namespace=ns) # Create test resources # Both users have access to the model via auth policy _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[user_principal, other_principal]) # Each user has their own subscription _create_test_subscription(user_subscription, MODEL_REF, users=[user_principal]) _create_test_subscription(other_subscription, MODEL_REF, users=[other_principal]) _wait_reconcile() # Test: User tries to use another user's subscription in header # Expected: 403 with "access denied" error log.info(f"Testing: GET /v1/models with inaccessible subscription: {other_subscription}") url = f"{_maas_api_url()}/v1/models" r = _request_with_gateway_retry( requests.get, url, headers={ "Authorization": f"Bearer {oc_token_user}", "x-maas-subscription": other_subscription, }, ) > assert r.status_code == 403, f"Expected 403 for inaccessible subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for inaccessible subscription, got 200: {"data":[{"id":"facebook/opt-125m","created":1783571959,"object":"model","owned_by":"llm/facebook-opt-125m-simulated","kind":"LLMInferenceService","url":"https://maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local/llm/facebook-opt-125m-simulated","ready":true,"modelDetails":{"description":"A simulated OPT-125M model for free-tier testing","displayName":"Facebook OPT 125M (Simulated)"},"subscriptions":[{"name":"simulator-subscription","displayName":"Simulator Subscription (Free)","description":"Free-tier subscription with 100 tokens/min rate limit"},{"name":"e2e-models-user-sub"}]}],"object":"list"} E assert 200 == 403 E + where 200 = <Response [200]>.status_code test/e2e/tests/test_models_endpoint.py:1687: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6d4072e2b0> def test_service_account_token_multiple_subs_with_header(self): """ Test: K8s token with access to multiple subscriptions filters by header. Creates a service account with access to two subscriptions. When querying with x-maas-subscription header, should return models from only the specified subscription. Expected: HTTP 200 with models from only the specified subscription. """ sa_name = "e2e-sa-multi-subs-with-header" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-sa-multi-hdr-sub1" sub2_name = "e2e-sa-multi-hdr-sub2" auth1_name = "e2e-sa-multi-hdr-auth1" auth2_name = "e2e-sa-multi-hdr-auth2" try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth1_name) _wait_for_maas_auth_policy_phase(auth2_name) _wait_for_maas_subscription_phase(sub1_name) _wait_for_maas_subscription_phase(sub2_name) # Query with K8s token and header specifying sub1 log.info(f"Querying /v1/models with K8s token and header: {sub1_name}") r1 = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {sa_token}", "x-maas-subscription": sub1_name, }, ) assert r1.status_code == 200, f"Expected 200, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} # Should see only models from sub1 assert DISTINCT_MODEL_ID in model_ids1, f"Should see {DISTINCT_MODEL_ID} from {sub1_name}" > assert DISTINCT_MODEL_2_ID not in model_ids1, f"Should NOT see {DISTINCT_MODEL_2_ID} from {sub2_name}" E AssertionError: Should NOT see test/e2e-distinct-model-2 from e2e-sa-multi-hdr-sub2 E assert 'test/e2e-distinct-model-2' not in {'facebook/opt-125m', 'test/e2e-distinct-model', 'test/e2e-distinct-model-2'} test/e2e/tests/test_models_endpoint.py:2013: AssertionError