/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestAPIKeyManagementIsolation object at 0x7f050aaea340> def test_api_key_cannot_mint_another_api_key(self): """A valid subscription-bound API key must be denied on POST /v1/api-keys. The initial key is minted with a cluster token as a control. The test then presents that valid key to the management endpoint and verifies that the gateway or MaaS API rejects it without returning new key material. """ _wait_for_gateway_auth_enforced() oc_token = _get_cluster_token() parent_key_id = None try: parent = _create_api_key_raw( oc_token, name=f"e2e-delegation-parent-{uuid.uuid4().hex[:8]}", subscription=SIMULATOR_SUBSCRIPTION, ) assert parent.status_code in (200, 201), ( f"Control API key mint failed: {parent.status_code} " f"body_bytes={len(parent.content)}" ) parent_body = parent.json() parent_key_id = parent_body.get("id") parent_key = parent_body.get("key", "") assert parent_key.startswith("sk-oai-"), "Control mint did not return an API key" # Use the API key, not the cluster token, to attempt delegation. # Do not log the response body: a regression could return a live key. delegated = requests.post( f"{_maas_api_url()}/v1/api-keys", headers={ "Authorization": f"Bearer {parent_key}", "Content-Type": "application/json", }, json={ "name": f"e2e-delegation-child-{uuid.uuid4().hex[:8]}", "subscription": SIMULATOR_SUBSCRIPTION, }, timeout=TIMEOUT, verify=TLS_VERIFY, ) log.info( "API-key-authenticated key mint -> %s body_bytes=%d", delegated.status_code, len(delegated.content), ) > assert delegated.status_code in (401, 403), ( f"Expected API-key-authenticated mint to be denied, got " f"{delegated.status_code} body_bytes={len(delegated.content)}" ) E AssertionError: Expected API-key-authenticated mint to be denied, got 201 body_bytes=326 E assert 201 in (401, 403) E + where 201 = <Response [201]>.status_code test/e2e/tests/test_negative_security.py:123: AssertionErrorself = <test_api_keys.TestAPIKeyBulkOperations object at 0x7f69910cd2b0> api_keys_base_url = 'https://maas.apps.c11d0304-e0a5-4709-b419-13ceda2bba51.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImFPT3pVTnVFLTc0UHhoUk1mVXEybUo0OFhwYWIwbWtSM1VtMEhhMWFaYkEifQ.e...N77BlNAtYag-NJvoiEEOVxGhHGwbnnbqOZC4DJHCn2mVtluNm_pcfrfRFCU-yble6pu-Xt5-jFUI04PHg', 'Content-Type': 'application/json'} admin_headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImFPT3pVTnVFLTc0UHhoUk1mVXEybUo0OFhwYWIwbWtSM1VtMEhhMWFaYkEifQ.e...iPb7lKEfhpwT5m0N3S9HIGwdTXc7nSwmNDYIPfN4tfF-sL5E5h9RRli6sbxcAuipT4ImN0gozxz36yIuA', 'Content-Type': 'application/json'} def test_bulk_revoke_combined_user_and_subscription(self, api_keys_base_url: str, headers: dict, admin_headers: dict): """Admin can revoke keys for a specific user within a specific subscription. Negative control: a second user's keys in the same subscription must remain active. """ if not admin_headers: pytest.skip("ADMIN_OC_TOKEN not set") sub_name = f"e2e-combo-sub-{os.urandom(4).hex()}" ns = _ns() sa_name = f"e2e-combo-sa-{os.urandom(4).hex()}" sa2_name = f"e2e-combo-sa2-{os.urandom(4).hex()}" key_ids = [] key_ids_user2 = [] try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) sa_headers = {"Authorization": f"Bearer {oc_token}", "Content-Type": "application/json"} oc_token2 = _create_sa_token(sa2_name, namespace=MODEL_NAMESPACE) sa_user2 = _sa_to_user(sa2_name, namespace=MODEL_NAMESPACE) sa2_headers = {"Authorization": f"Bearer {oc_token2}", "Content-Type": "application/json"} > _create_test_auth_policy(f"{sub_name}-auth", MODEL_REF, users=[sa_user, sa_user2]) test/e2e/tests/test_api_keys.py:540: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-combo-sub-537e7727...system:serviceaccount:llm:e2e-combo-sa-a18a8657", "system:serviceaccount:llm:e2e-combo-sa2-6b8acdf4"], "groups": []}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...ontroller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6bbc425a90> def test_aitenant_create_bootstrap_resources(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) > _apply_aitenant(case) test/e2e/tests/test_aitenant_lifecycle.py:421: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:191: in _apply_aitenant _apply( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-ait-3b43a1da', 'namespace': 'ai-tenants'}, 'spec': {}} def _apply(obj): result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": EOF test/e2e/tests/test_aitenant_lifecycle.py:69: RuntimeError/workspace/source/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_auto_resolve.TestTenantAutoResolve object at 0x7f13d486e520> def test_auto_resolve_populates_resolved_tenant_ref(self): """When tenantRef is omitted, resolvedTenantRef is populated from the gateway.""" case = new_named_tenant_case("e2e-autores") model_name = f"autores-model-{case['suffix']}" try: > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_auto_resolve.py:91: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:783: in bootstrap_aitenant_tenant apply_aitenant(case) test/e2e/tests/multitenancy_helpers.py:742: in apply_aitenant _apply( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-autores-4454b7', 'namespace': 'ai-tenants'}, 'spec': {'gateway': {'name': 'e2e-autores-4454b7'}}} def _apply(obj: dict) -> None: result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": EOF test/e2e/tests/multitenancy_helpers.py:155: RuntimeErrorself = <test_negative_security.TestMissingModelRef object at 0x7f050aaf1670> def test_subscription_with_nonexistent_model_ref(self): """MaaSSubscription generates TRLP only for valid model, not ghost model. Creates a subscription referencing one valid model and one ghost model, waits for Degraded phase, then asserts that a TRLP exists for the valid model but not for the ghost model. """ suffix = uuid.uuid4().hex[:8] sub_name = f"e2e-neg-ghost-sub-{suffix}" auth_name = f"e2e-neg-ghost-sub-auth-{suffix}" ghost_model = f"nonexistent-model-{suffix}" try: > _create_test_auth_policy(auth_name, MODEL_REF, groups=["system:authenticated"]) test/e2e/tests/test_negative_security.py:514: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-neg-ghost-sub-auth...k-opt-125m-simulated", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...ontroller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_negative_security.TestMissingModelRef object at 0x7f050aaf18b0> def test_authpolicy_with_nonexistent_model_ref(self): """MaaSAuthPolicy does not generate per-model AuthPolicies in gateway-only mode. Creates an auth policy referencing one valid model and one ghost model, waits for Degraded phase, then asserts that no legacy per-model AuthPolicies exist for either model. """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-ghost-policy-{suffix}" ghost_model = f"nonexistent-model-{suffix}" try: > _create_test_auth_policy( policy_name, [MODEL_REF, ghost_model], groups=["system:authenticated"], ) test/e2e/tests/test_negative_security.py:555: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-neg-ghost-policy-3...stent-model-332227cd", "namespace": "llm"}], "subjects": {"users": [], "groups": [{"name": "system:authenticated"}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...3/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": http: server gave HTTP response to HTTPS client\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6bbc4bafa0> def test_aitenant_migrates_and_removes_legacy_tenant(self): if not _crd_exists(LEGACY_TENANT_CRD): pytest.skip(f"Missing CRD {LEGACY_TENANT_CRD}; legacy Tenant migration test is not applicable") suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-migrate-{suffix}" tenant_ns = f"ai-tenant-{aitenant_name}" gateway_name = f"{aitenant_name}-gw" max_expiration_days = 37 try: _apply({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": tenant_ns}}) _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "Tenant", "metadata": { "name": TENANT_NAME, "namespace": tenant_ns, }, "spec": { "gatewayRef": { "namespace": GATEWAY_NAMESPACE, "name": gateway_name, }, "apiKeys": { "maxExpirationDays": max_expiration_days, }, "externalOIDC": { "issuerUrl": "https://issuer.example.com/realms/e2e", "clientId": "e2e-client", "ttl": 600, }, "telemetry": { "enabled": False, "metrics": { "captureOrganization": False, "captureUser": True, "captureGroup": True, "captureModelUsage": False, }, }, }, } ) > _apply_aitenant( { "aitenant_name": aitenant_name, "tenant_ns": tenant_ns, "gateway_name": gateway_name, } ) test/e2e/tests/test_aitenant_lifecycle.py:472: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:191: in _apply_aitenant _apply( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-migrate-fbdf5841', 'namespace': 'ai-tenants'}, 'spec': {}} def _apply(obj): result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/e2e/tests/test_aitenant_lifecycle.py:69: RuntimeErrorself = <test_negative_security.TestWebhookValidation object at 0x7f050aaf1e50> def test_subscription_rejected_in_unlabeled_namespace(self): """MaaSSubscription create is rejected in namespace without MaasTenantConfig CR. Webhooks require namespaces to have a MaasTenantConfig CR to contain tenant resources. """ test_ns = f"e2e-webhook-test-{uuid.uuid4().hex[:6]}" try: # Create namespace without MaasTenantConfig CR result = subprocess.run( ["oc", "create", "namespace", test_ns], capture_output=True, text=True, timeout=30 ) assert result.returncode == 0, f"Failed to create namespace: {result.stderr}" # Try to create MaaSSubscription (should be rejected by webhook) result = subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "test-sub", "namespace": test_ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}] }], }, }), capture_output=True, text=True, timeout=30 ) # Verify webhook rejection assert result.returncode != 0, "Expected webhook to reject subscription in namespace without MaasTenantConfig CR" > assert "admission webhook" in result.stderr.lower(), \ f"Expected webhook rejection, got: {result.stderr}" E AssertionError: Expected webhook rejection, got: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaassubscription.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service" E E assert 'admission webhook' in 'error from server (internalerror): error when creating "stdin": internal error occurred: failed calling webhook "vmaassubscription.kb.io": failed to call webhook: post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' E + where 'error from server (internalerror): error when creating "stdin": internal error occurred: failed calling webhook "vmaassubscription.kb.io": failed to call webhook: post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' = <built-in method lower of str object at 0x7f050a9ff510>() E + where <built-in method lower of str object at 0x7f050a9ff510> = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaassubscription.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n'.lower E + where 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaassubscription.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' = CompletedProcess(args=['oc', 'apply', '-f', '-'], returncode=1, stdout='', stderr='Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaassubscription.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maassubscription?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n').stderr test/e2e/tests/test_negative_security.py:657: AssertionErrorself = <test_tenant_auto_resolve.TestTenantAutoResolve object at 0x7f13d4887850> def test_explicit_tenant_ref_preserved(self): """When tenantRef is set explicitly, resolvedTenantRef reflects the explicit value.""" case = new_named_tenant_case("e2e-explicit") model_name = f"explicit-model-{case['suffix']}" try: > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_auto_resolve.py:127: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:783: in bootstrap_aitenant_tenant apply_aitenant(case) test/e2e/tests/multitenancy_helpers.py:742: in apply_aitenant _apply( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-explicit-bae22d', 'namespace': 'ai-tenants'}, 'spec': {'gateway': {'name': 'e2e-explicit-bae22d'}}} def _apply(obj: dict) -> None: result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/e2e/tests/multitenancy_helpers.py:155: RuntimeErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6bbc3d95b0> def test_aitenant_delete_cleans_maas_resources_and_preserves_user_objects(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) > _apply_aitenant(case) test/e2e/tests/test_aitenant_lifecycle.py:541: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:191: in _apply_aitenant _apply( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-ait-f7fc2b9f', 'namespace': 'ai-tenants'}, 'spec': {}} def _apply(obj): result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/e2e/tests/test_aitenant_lifecycle.py:69: RuntimeErrorself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f69910d6490> def test_create_key_for_active_subscription(self): """API key creation succeeds for Active subscription.""" ns = _ns() subscription_name = "e2e-apikey-active-sub" auth_name = "e2e-apikey-active-auth" sa_name = "e2e-apikey-active-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) > _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) test/e2e/tests/test_api_keys.py:1398: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-apikey-active-auth...ated", "namespace": "llm"}], "subjects": {"users": ["system:serviceaccount:llm:e2e-apikey-active-sa"], "groups": []}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f69910d6a90> def test_create_key_for_degraded_subscription(self): """API key creation succeeds for Degraded subscription.""" ns = _ns() subscription_name = "e2e-apikey-degraded-sub" auth_name = "e2e-apikey-degraded-auth" sa_name = "e2e-apikey-degraded-sa" missing_model = "nonexistent-model-apikey" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) > _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) test/e2e/tests/test_api_keys.py:1435: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-apikey-degraded-au...ed", "namespace": "llm"}], "subjects": {"users": ["system:serviceaccount:llm:e2e-apikey-degraded-sa"], "groups": []}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f69910d67c0> def test_create_key_for_failed_subscription(self): """API key creation is rejected for Failed subscription to prevent key spam.""" ns = _ns() subscription_name = "e2e-apikey-failed-sub" auth_name = "e2e-apikey-failed-auth" sa_name = "e2e-apikey-failed-sa" nonexistent_model = "nonexistent-model-apikey-failed" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) > _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) test/e2e/tests/test_api_keys.py:1477: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:908: in _create_test_auth_policy _apply_cr({ test/e2e/tests/test_helper.py:620: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "e2e-apikey-failed-auth...ated", "namespace": "llm"}], "subjects": {"users": ["system:serviceaccount:llm:e2e-apikey-failed-sa"], "groups": []}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaa...atahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorself = <test_negative_security.TestWebhookValidation object at 0x7f050aaf90d0> def test_authpolicy_rejected_in_unlabeled_namespace(self): """MaaSAuthPolicy create is rejected in namespace without MaasTenantConfig CR.""" test_ns = f"e2e-webhook-test-{uuid.uuid4().hex[:6]}" try: # Create namespace without MaasTenantConfig CR result = subprocess.run( ["oc", "create", "namespace", test_ns], capture_output=True, text=True, timeout=30 ) assert result.returncode == 0, f"Failed to create namespace: {result.stderr}" # Try to create MaaSAuthPolicy (should be rejected by webhook) result = subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "test-policy", "namespace": test_ns}, "spec": { "modelRefs": [{"name": MODEL_REF, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }), capture_output=True, text=True, timeout=30 ) # Verify webhook rejection assert result.returncode != 0, "Expected webhook to reject auth policy in namespace without MaasTenantConfig CR" > assert "admission webhook" in result.stderr.lower(), \ f"Expected webhook rejection, got: {result.stderr}" E AssertionError: Expected webhook rejection, got: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service" E E assert 'admission webhook' in 'error from server (internalerror): error when creating "stdin": internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' E + where 'error from server (internalerror): error when creating "stdin": internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' = <built-in method lower of str object at 0x7f050a9ff6b0>() E + where <built-in method lower of str object at 0x7f050a9ff6b0> = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n'.lower E + where 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n' = CompletedProcess(args=['oc', 'apply', '-f', '-'], returncode=1, stdout='', stderr='Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": no endpoints available for service "maas-controller-webhook-service"\n').stderr test/e2e/tests/test_negative_security.py:703: AssertionErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6bbc3d9430> def test_aitenant_derives_non_default_tenant_namespace(self): """RHOAIENG-66836: non-default AITenant must not use models-as-a-service tenant namespace.""" suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-derive-{suffix}" reserved_ns = _ns() expected_ns = f"ai-tenant-{aitenant_name}" gateway_name = aitenant_name try: _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) > _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "AITenant", "metadata": {"name": aitenant_name, "namespace": AITENANT_NAMESPACE}, "spec": {}, } ) test/e2e/tests/test_aitenant_lifecycle.py:602: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-derive-79e15b7c', 'namespace': 'ai-tenants'}, 'spec': {}} def _apply(obj): result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/e2e/tests/test_aitenant_lifecycle.py:69: RuntimeError/workspace/source/test/e2e/tests/test_external_oidc.py:252: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:263: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:277: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:290: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:303: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:335: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:379: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:395: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:405: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:428: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:436: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:441: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:454: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:501: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:537: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:602: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:633: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:659: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:690: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:750: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:786: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:961: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1013: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1049: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1103: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1142: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_subscription.py:1069: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:189: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:355: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:215: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B