/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestE2ESubscriptionFlow object at 0x7ff8de6c1fd0> def test_e2e_mint_api_key_denied_for_inaccessible_subscription(self): """POST /v1/api-keys with another user's subscription returns generic invalid_subscription.""" ns = _ns() auth_policy_name = "e2e-test-auth-access-denied" user_subscription = "e2e-test-user-subscription" other_subscription = "e2e-test-other-subscription" sa_user = "e2e-sa-user" sa_other = "e2e-sa-other" try: # Create two service accounts oc_token_user = _create_sa_token(sa_user, namespace=ns) _ = _create_sa_token(sa_other, namespace=ns) # SA creation only - token unused user_principal = _sa_to_user(sa_user, namespace=ns) other_principal = _sa_to_user(sa_other, namespace=ns) # Create test resources # Both users have access to the model > _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[user_principal, other_principal]) test/e2e/tests/test_subscription.py:1560: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:511: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:279: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-test-auth-access-d...ceaccount:models-as-a-service:e2e-sa-user", "system:serviceaccount:models-as-a-service:e2e-sa-other"], "groups": []}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...ontroller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7ff8deb99a60> def test_e2e_group_based_access_gets_200(self): """ E2E test: Group-based auth and subscription (success case). Validates that users can access models via group membership in both MaaSAuthPolicy and MaaSSubscription, not just explicit user lists. """ ns = _ns() auth_policy_name = "e2e-test-group-auth" subscription_name = "e2e-test-group-subscription" sa_name = "e2e-sa-group" # Use namespace-specific group that SA will be in test_group = f"system:serviceaccounts:{ns}" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) # Create auth policy using GROUP (not user) > _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group]) test/e2e/tests/test_subscription.py:1621: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:511: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:279: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-test-group-auth", ... "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:serviceaccounts:models-as-a-service"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7ff8deb99640> def test_e2e_group_based_auth_but_no_subscription_gets_403(self): """ E2E test: Group-based auth, but user's group not in any subscription (failure case). Validates that having auth via group membership is not sufficient if the user's groups don't match any subscription's owner groups. """ ns = _ns() auth_policy_name = "e2e-test-group-auth-only" sa_name = "e2e-sa-group-auth-only" # Use namespace-specific group for auth test_group = f"system:serviceaccounts:{ns}" # Snapshot existing subscription to restore later original_sim = _snapshot_cr("maassubscription", SIMULATOR_SUBSCRIPTION) try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) # Create auth policy using group _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group]) > api_key = _create_api_key( oc_token, name=f"{sa_name}-key", subscription=SIMULATOR_SUBSCRIPTION, ) test/e2e/tests/test_subscription.py:1669: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6Im1Gb3M0dUtRaVF1amNLOGdMYjZLQzZRRzMtZDdoQXpsYzJ6TWFCYWlvLVEifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm...POtDqJtmNkrCRUCKGZhydXVyX6sbz8eKCompxfqWAGS0abUOWgEtHNSIPR-wzvloLMcFki8OCVYe-q8-2RvAHXrJu9eOW3mGqZzXfhjXxtGUcFeY31MDCA' name = 'e2e-sa-group-auth-only-key', subscription = 'simulator-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 500 {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"001"} test/e2e/tests/test_helper.py:247: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7ff8de6a7130> def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid model (will be Active) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Verify it starts as Active cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") log.info(f"Initial phase: {phase}") assert phase == "Active", f"Expected Active initially, got {phase}" # Create API key while Active api_key = _create_api_key( oc_token, name="failed-sub-test", subscription=subscription_name ) # Verify inference works while Active log.info("Testing inference while Active...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) assert r.status_code == 200, f"Expected 200 while Active, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference works with Active subscription") # Manually patch subscription to Failed phase import subprocess import json from datetime import datetime log.info("Manually patching subscription to Failed phase...") patch_data = { "status": { "phase": "Failed", "conditions": [ { "type": "Ready", "status": "False", "reason": "Failed", "message": "Subscription failed", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") } ], "modelRefStatuses": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Model failed" } ] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch to Failed phase: {result.stderr}" # Verify phase is Failed cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") > assert phase == "Failed", f"Expected Failed phase after patch, got {phase}" E AssertionError: Expected Failed phase after patch, got Active E assert 'Active' == 'Failed' E E - Failed E + Active test/e2e/tests/test_subscription.py:2357: AssertionErrorself = <Response [403]>, kwargs = {} def json(self, **kwargs): r"""Decodes the JSON response body (if any) as a Python object. This may return a dictionary, list, etc. depending on what is in the response. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. :raises requests.exceptions.JSONDecodeError: If the response body does not contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: try: return complexjson.loads(self.content.decode(encoding), **kwargs) except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, # and the server didn't bother to tell us what codec *was* # used. pass except JSONDecodeError as e: raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) try: > return complexjson.loads(self.text, **kwargs) test/e2e/.venv/lib64/python3.9/site-packages/requests/models.py:976: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.9/json/__init__.py:346: in loads return _default_decoder.decode(s) /usr/lib64/python3.9/json/decoder.py:337: in decode obj, end = self.raw_decode(s, idx=_w(s, 0).end()) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <json.decoder.JSONDecoder object at 0x7ff8dff72760>, s = '', idx = 0 def raw_decode(self, s, idx=0): """Decode a JSON document from ``s`` (a ``str`` beginning with a JSON document) and return a 2-tuple of the Python representation and the index in ``s`` where the document ended. This can be used to decode a JSON document from a string that may have extraneous data at the end. """ try: obj, end = self.scan_once(s, idx) except StopIteration as err: > raise JSONDecodeError("Expecting value", s, err.value) from None E json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) /usr/lib64/python3.9/json/decoder.py:355: JSONDecodeError During handling of the above exception, another exception occurred: self = <test_models_endpoint.TestModelsEndpoint object at 0x7ff8deb8b9a0> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403 with error type: permission_error """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_reconcile() # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_reconcile() # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" > data = r.json() test/e2e/tests/test_models_endpoint.py:1506: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Response [403]>, kwargs = {} def json(self, **kwargs): r"""Decodes the JSON response body (if any) as a Python object. This may return a dictionary, list, etc. depending on what is in the response. :param \*\*kwargs: Optional arguments that ``json.loads`` takes. :raises requests.exceptions.JSONDecodeError: If the response body does not contain valid json. """ if not self.encoding and self.content and len(self.content) > 3: # No encoding set. JSON RFC 4627 section 3 states we should expect # UTF-8, -16 or -32. Detect which one to use; If the detection or # decoding fails, fall back to `self.text` (using charset_normalizer to make # a best guess). encoding = guess_json_utf(self.content) if encoding is not None: try: return complexjson.loads(self.content.decode(encoding), **kwargs) except UnicodeDecodeError: # Wrong UTF codec detected; usually because it's not UTF-8 # but some other 8-bit codec. This is an RFC violation, # and the server didn't bother to tell us what codec *was* # used. pass except JSONDecodeError as e: raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) try: return complexjson.loads(self.text, **kwargs) except JSONDecodeError as e: # Catch JSON-related errors and raise as requests.JSONDecodeError # This aliases json.JSONDecodeError and simplejson.JSONDecodeError > raise RequestsJSONDecodeError(e.msg, e.doc, e.pos) E requests.exceptions.JSONDecodeError: Expecting value: line 1 column 1 (char 0) test/e2e/.venv/lib64/python3.9/site-packages/requests/models.py:980: JSONDecodeError