/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_api_keys.TestAPIKeySubscriptionFilter object at 0x7f5d661451d0> api_keys_base_url = 'https://e2e-worker-w2-bb4ecb.apps.df465f239a4f81e65420.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im1QdVNLQTR6c1FjYnVYZm9pbzFsajB4LTZXdkxrQjhJNzgwdGpwdFc4M2MifQ.e...nl-_-zjt7SinO2EVBlW-qaYOyKKHLxQEYyd2cdLFH3yq-uN0bs7esSZwMLM5Tvp-p2XwWMuLyU58bu7rA', 'Content-Type': 'application/json'} def test_search_filters_by_subscription(self, api_keys_base_url: str, headers: dict): """Search with subscription filter returns only keys bound to that subscription.""" sub_a = f"e2e-filter-sub-a-{os.urandom(4).hex()}" sub_b = f"e2e-filter-sub-b-{os.urandom(4).hex()}" ns = _ns() sa_name = f"e2e-filter-sa-{os.urandom(4).hex()}" key_ids_a = [] key_ids_b = [] try: # Create one SA authorized for both subscriptions so that # exclusion in search results is attributable to the subscription # filter, not user-scoping. > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/maas-e2e/test/e2e/tests/test_api_keys.py:1831: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-filter-sa-2dc914d7' namespace = 'e2e-models-e2e-worker-w2-bb4ecb', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") result = subprocess.run( ["oc", "create", "token", sa_name, "-n", namespace, f"--duration={duration}"], capture_output=True, text=True, ) token = result.stdout.strip() if not token: > raise RuntimeError(f"Could not create token for SA {sa_name}: {result.stderr}") E RuntimeError: Could not create token for SA e2e-filter-sa-2dc914d7: error: failed to create token: Post "https://ae5958bb2383e433ca187ce3cb02f6df-5b08594c4817760f.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/e2e-models-e2e-worker-w2-bb4ecb/serviceaccounts/e2e-filter-sa-2dc914d7/token": net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/test_helper.py:476: RuntimeErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7fb4cb677690> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA > sa_token = _create_sa_token(sa_name, namespace=sa_ns) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/maas-e2e/test/e2e/tests/test_models_endpoint.py:1964: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-multi-keys-sa', namespace = 'default', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-multi-keys-sa: error: failed to create serviceaccount: Post "https://ae5958bb2383e433ca187ce3cb02f6df-5b08594c4817760f.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/default/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/test_helper.py:469: RuntimeErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f826fb340d0> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403. The gateway may reject the revoked key before the request reaches maas-api, in which case the response body is empty. """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_subscription_discovery_ready(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_for_cr_absent("maassubscription", subscription_name, namespace=ns) # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist > assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for API key with deleted subscription, got 500: Internal Server Error. E E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/maas-e2e/test/e2e/tests/test_models_endpoint.py:1643: AssertionErrormaas_api_base_url = 'https://maas.apps.df465f239a4f81e65420.prod.konflux-ocp-ci.dev/maas-api' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im1QdVNLQTR6c1FjYnVYZm9pbzFsajB4LTZXdkxrQjhJNzgwdGpwdFc4M2MifQ.e...nl-_-zjt7SinO2EVBlW-qaYOyKKHLxQEYyd2cdLFH3yq-uN0bs7esSZwMLM5Tvp-p2XwWMuLyU58bu7rA', 'Content-Type': 'application/json'} @pytest.fixture(scope="session") def model_catalog(maas_api_base_url: str, headers: dict): r = requests.get(f"{maas_api_base_url}/v1/models", headers=headers, timeout=45, verify=TLS_VERIFY) > r.raise_for_status() test/maas-e2e/test/e2e/tests/conftest.py:289: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Response [500]> def raise_for_status(self) -> None: """Raises :class:`HTTPError`, if one occurred.""" http_error_msg = "" if isinstance(self.reason, bytes): # We attempt to decode utf-8 first because some servers # choose to localize their reason strings. If the string # isn't utf-8, we fall back to iso-8859-1 for all other # encodings. (See PR #3538) try: reason = self.reason.decode("utf-8") except UnicodeDecodeError: reason = self.reason.decode("iso-8859-1") else: reason = self.reason if 400 <= self.status_code < 500: http_error_msg = ( f"{self.status_code} Client Error: {reason} for url: {self.url}" ) elif 500 <= self.status_code < 600: http_error_msg = ( f"{self.status_code} Server Error: {reason} for url: {self.url}" ) if http_error_msg: > raise HTTPError(http_error_msg, response=self) E requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://maas.apps.df465f239a4f81e65420.prod.konflux-ocp-ci.dev/maas-api/v1/models test/e2e/.venv/lib64/python3.11/site-packages/requests/models.py:1167: HTTPErrormaas_api_base_url = 'https://maas.apps.df465f239a4f81e65420.prod.konflux-ocp-ci.dev/maas-api' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Im1QdVNLQTR6c1FjYnVYZm9pbzFsajB4LTZXdkxrQjhJNzgwdGpwdFc4M2MifQ.e...nl-_-zjt7SinO2EVBlW-qaYOyKKHLxQEYyd2cdLFH3yq-uN0bs7esSZwMLM5Tvp-p2XwWMuLyU58bu7rA', 'Content-Type': 'application/json'} @pytest.fixture(scope="session") def model_catalog(maas_api_base_url: str, headers: dict): r = requests.get(f"{maas_api_base_url}/v1/models", headers=headers, timeout=45, verify=TLS_VERIFY) > r.raise_for_status() test/maas-e2e/test/e2e/tests/conftest.py:289: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Response [500]> def raise_for_status(self) -> None: """Raises :class:`HTTPError`, if one occurred.""" http_error_msg = "" if isinstance(self.reason, bytes): # We attempt to decode utf-8 first because some servers # choose to localize their reason strings. If the string # isn't utf-8, we fall back to iso-8859-1 for all other # encodings. (See PR #3538) try: reason = self.reason.decode("utf-8") except UnicodeDecodeError: reason = self.reason.decode("iso-8859-1") else: reason = self.reason if 400 <= self.status_code < 500: http_error_msg = ( f"{self.status_code} Client Error: {reason} for url: {self.url}" ) elif 500 <= self.status_code < 600: http_error_msg = ( f"{self.status_code} Server Error: {reason} for url: {self.url}" ) if http_error_msg: > raise HTTPError(http_error_msg, response=self) E requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://maas.apps.df465f239a4f81e65420.prod.konflux-ocp-ci.dev/maas-api/v1/models test/e2e/.venv/lib64/python3.11/site-packages/requests/models.py:1167: HTTPError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy