/workspace/source/test/e2e/tests/test_api_keys.py:930: CronJob maas-api-key-cleanup not found in opendatahub: Error from server (NotFound): cronjobs.batch "maas-api-key-cleanup" not found/workspace/source/test/e2e/tests/test_api_keys.py:977: NetworkPolicy maas-api-cleanup-restrict not found in opendatahub: Error from server (NotFound): networkpolicies.networking.k8s.io "maas-api-cleanup-restrict" not found/workspace/source/test/e2e/tests/test_api_keys.py:1096: Cannot find maas-api pod in opendatahub: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdself = <Response [403]>, kwargs = {} def json(self, **kwargs): r"""Decodes the JSON response body (if any) as a Python object. This may return a dictionary, list, etc. depending on what is in the response. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. :raises requests.exceptions.JSONDecodeError: If the response body does not contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: try: return complexjson.loads(self.content.decode(encoding), **kwargs) except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, # and the server didn't bother to tell us what codec *was* # used. pass except JSONDecodeError as e: raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) try: > return complexjson.loads(self.text, **kwargs) test/e2e/.venv/lib64/python3.9/site-packages/requests/models.py:976: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.9/json/__init__.py:346: in loads return _default_decoder.decode(s) /usr/lib64/python3.9/json/decoder.py:337: in decode obj, end = self.raw_decode(s, idx=_w(s, 0).end()) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <json.decoder.JSONDecoder object at 0x7fc3f0364760>, s = '', idx = 0 def raw_decode(self, s, idx=0): """Decode a JSON document from ``s`` (a ``str`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. """ try: obj, end = self.scan_once(s, idx) except StopIteration as err: > raise JSONDecodeError("Expecting value", s, err.value) from None E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) /usr/lib64/python3.9/json/decoder.py:355: JSONDecodeError During handling of the above exception, another exception occurred: self = <test_models_endpoint.TestModelsEndpoint object at 0x7fc3eefa9610> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403 with error type: permission_error """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_reconcile() # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_reconcile() # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" > data = r.json() test/e2e/tests/test_models_endpoint.py:1506: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Response [403]>, kwargs = {} def json(self, **kwargs): r"""Decodes the JSON response body (if any) as a Python object. This may return a dictionary, list, etc. depending on what is in the response. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. :raises requests.exceptions.JSONDecodeError: If the response body does not contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: try: return complexjson.loads(self.content.decode(encoding), **kwargs) except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, # and the server didn't bother to tell us what codec *was* # used. pass except JSONDecodeError as e: raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) try: return complexjson.loads(self.text, **kwargs) except JSONDecodeError as e: # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError > raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) E requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0) test/e2e/.venv/lib64/python3.9/site-packages/requests/models.py:980: JSONDecodeError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsself = <test_tenant_model_inference.TestTenantModelInference object at 0x7fc3eed7aca0> def test_tenant_deletion_prevents_key_reuse_on_recreation(self): """Tenant deletion revokes keys - orphaned keys don't work if tenant is recreated. Security test (RHOAIENG-72792): When a tenant is deleted and recreated with the same name, old API keys from the deleted tenant should not grant access. Steps: 1. Create AITenant 2. Create API key 3. Delete AITenant 4. Recreate AITenant with same name 5. Verify old API key does not work """ require_aitenant_crd() case = new_named_tenant_case("e2e-reuse") orphaned_key = None try: # Step 1: Create AITenant bootstrap_aitenant_tenant(case) model_name = f"test-model-{case['suffix']}" _create_llmis(model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE) wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180 ) _create_maas_model_ref(model_name, case["tenant_ns"], model_name) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"] ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"] ) wait_for_status_phase("maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180) # Step 2: Create API key gateway_url = _get_tenant_gateway_url(case["gateway_name"]) oc_token = _get_cluster_token() r = requests.post( f"{gateway_url}/maas-api/v1/api-keys", headers={"Authorization": f"Bearer {oc_token}", "Content-Type": "application/json"}, json={"name": "e2e-orphaned-key", "subscription": f"{model_name}-sub"}, timeout=45, verify=TLS_VERIFY, ) > assert r.status_code in (200, 201), f"Failed to create key: {r.status_code}" E AssertionError: Failed to create key: 503 E assert 503 in (200, 201) E + where 503 = <Response [503]>.status_code test/e2e/tests/test_tenant_model_inference.py:328: AssertionErrorgateway_host = 'maas.apps.33c7548f-6bf5-4599-8c33-58219654a39a.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-4ea2af' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-4ea2af in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionErrorgateway_host = 'maas.apps.33c7548f-6bf5-4599-8c33-58219654a39a.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-4ea2af' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-4ea2af in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionErrorgateway_host = 'maas.apps.33c7548f-6bf5-4599-8c33-58219654a39a.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-4ea2af' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-4ea2af in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionError