/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestHeaderSpoofing object at 0x7f18bf395ee0> def test_duplicate_subscription_headers_ignored(self): """Client sends multiple X-MaaS-Subscription headers — API key binding wins. For API key requests, the subscription is fixed at mint time. Duplicate or conflicting X-MaaS-Subscription headers must not override the key-derived subscription. """ _wait_for_gateway_auth_enforced() api_key = _create_api_key(_get_cluster_token(), subscription=SIMULATOR_SUBSCRIPTION) # Warm up: confirm the API key works with a normal request before # testing duplicate headers. Under parallel load, Rego policy # propagation can take longer than the 30s retry window below. _poll_status(api_key, 200, timeout=60) # Use http.client to send genuinely duplicate X-MaaS-Subscription headers. # The requests library uses a dict for headers, so it cannot send two # headers with the same name — the second value overwrites the first. path = f"{MODEL_PATH}/v1/completions" body = json.dumps({"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 3}) # Two separate X-MaaS-Subscription header lines headers = [ ("Authorization", f"Bearer {api_key}"), ("Content-Type", "application/json"), ("X-MaaS-Subscription", SIMULATOR_SUBSCRIPTION), ("X-MaaS-Subscription", "nonexistent-fake-sub"), ] # As above, keep the raw duplicate headers on every retry while waiting # for shared gateway authorization state to settle. deadline = time.time() + 30 while True: gateway = _gateway_url() parsed = urlparse(gateway) if parsed.scheme == "https": ctx = ssl.create_default_context() if not TLS_VERIFY: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE conn = http.client.HTTPSConnection( parsed.hostname, parsed.port or 443, timeout=TIMEOUT, context=ctx, ) else: conn = http.client.HTTPConnection( parsed.hostname, parsed.port or 80, timeout=TIMEOUT, ) try: conn.putrequest("POST", path) for key, value in headers: conn.putheader(key, value) conn.putheader("Content-Length", str(len(body))) conn.endheaders(body.encode()) resp = conn.getresponse() status = resp.status resp_body = resp.read().decode(errors="replace") finally: conn.close() if status == 200 or time.time() >= deadline: break time.sleep(2) # API key binding wins — request succeeds with key-derived subscription. log.info("Duplicate X-MaaS-Subscription headers -> %s", status) > assert status == 200, ( f"Expected 200 (API key subscription binding wins over duplicate headers), " f"got {status}: {resp_body[:500]}" ) E AssertionError: Expected 200 (API key subscription binding wins over duplicate headers), got 403: E assert 403 == 200 test/e2e/tests/test_negative_security.py:335: AssertionError/workspace/source/test/e2e/tests/test_tenant_discovery.py:136: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:212: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:252: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:263: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:277: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:290: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:303: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:335: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:379: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:395: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:405: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:428: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:436: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:441: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:454: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:501: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:537: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:602: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:633: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:659: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:690: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:750: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:786: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:961: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1013: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1049: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1103: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1142: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_subscription.py:1069: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:189: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:355: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueshared_test_tenants = ({'base_url': 'https://e2e-shared-a-w3-480517.apps.74e2d130-d209-4757-8718-dc69419b1309.prod.konfluxeaas.com/maas-api'...me': 'e2e-shared-b-w3-17e67b', 'name': 'e2e-shared-b-w3-17e67b', 'namespace': 'ai-tenant-e2e-shared-b-w3-17e67b', ...}) @pytest.fixture(scope="module") def tenant_env(shared_test_tenants): """Adapter fixture with tenant-specific models.""" case_a, case_b = dict(shared_test_tenants[0]), dict(shared_test_tenants[1]) for case in (case_a, case_b): model_name = f"auth-test-model-{case['suffix']}" > provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) test/e2e/tests/test_tenant_auth_isolation.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:904: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/e2e/tests/test_helper.py:1490: in _create_llmis _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceService", "metadata": {"name": "auth-test-model-480...ttps", "scheme": "HTTPS"}}, "readinessProbe": {"httpGet": {"path": "/ready", "port": "https", "scheme": "HTTPS"}}}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "llmi...webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceservice?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorshared_test_tenants = ({'base_url': 'https://e2e-shared-a-w3-480517.apps.74e2d130-d209-4757-8718-dc69419b1309.prod.konfluxeaas.com/maas-api'...me': 'e2e-shared-b-w3-17e67b', 'name': 'e2e-shared-b-w3-17e67b', 'namespace': 'ai-tenant-e2e-shared-b-w3-17e67b', ...}) @pytest.fixture(scope="module") def tenant_env(shared_test_tenants): """Adapter fixture with tenant-specific models.""" case_a, case_b = dict(shared_test_tenants[0]), dict(shared_test_tenants[1]) for case in (case_a, case_b): model_name = f"auth-test-model-{case['suffix']}" > provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) test/e2e/tests/test_tenant_auth_isolation.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:904: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/e2e/tests/test_helper.py:1490: in _create_llmis _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceService", "metadata": {"name": "auth-test-model-480...ttps", "scheme": "HTTPS"}}, "readinessProbe": {"httpGet": {"path": "/ready", "port": "https", "scheme": "HTTPS"}}}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "llmi...webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceservice?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorshared_test_tenants = ({'base_url': 'https://e2e-shared-a-w3-480517.apps.74e2d130-d209-4757-8718-dc69419b1309.prod.konfluxeaas.com/maas-api'...me': 'e2e-shared-b-w3-17e67b', 'name': 'e2e-shared-b-w3-17e67b', 'namespace': 'ai-tenant-e2e-shared-b-w3-17e67b', ...}) @pytest.fixture(scope="module") def tenant_env(shared_test_tenants): """Adapter fixture with tenant-specific models.""" case_a, case_b = dict(shared_test_tenants[0]), dict(shared_test_tenants[1]) for case in (case_a, case_b): model_name = f"auth-test-model-{case['suffix']}" > provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) test/e2e/tests/test_tenant_auth_isolation.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:904: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/e2e/tests/test_helper.py:1490: in _create_llmis _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceService", "metadata": {"name": "auth-test-model-480...ttps", "scheme": "HTTPS"}}, "readinessProbe": {"httpGet": {"path": "/ready", "port": "https", "scheme": "HTTPS"}}}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "llmi...webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceservice?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorshared_test_tenants = ({'base_url': 'https://e2e-shared-a-w3-480517.apps.74e2d130-d209-4757-8718-dc69419b1309.prod.konfluxeaas.com/maas-api'...me': 'e2e-shared-b-w3-17e67b', 'name': 'e2e-shared-b-w3-17e67b', 'namespace': 'ai-tenant-e2e-shared-b-w3-17e67b', ...}) @pytest.fixture(scope="module") def tenant_env(shared_test_tenants): """Adapter fixture with tenant-specific models.""" case_a, case_b = dict(shared_test_tenants[0]), dict(shared_test_tenants[1]) for case in (case_a, case_b): model_name = f"auth-test-model-{case['suffix']}" > provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) test/e2e/tests/test_tenant_auth_isolation.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:904: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/e2e/tests/test_helper.py:1490: in _create_llmis _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceService", "metadata": {"name": "auth-test-model-480...ttps", "scheme": "HTTPS"}}, "readinessProbe": {"httpGet": {"path": "/ready", "port": "https", "scheme": "HTTPS"}}}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "llmi...webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceservice?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessErrorshared_test_tenants = ({'base_url': 'https://e2e-shared-a-w3-480517.apps.74e2d130-d209-4757-8718-dc69419b1309.prod.konfluxeaas.com/maas-api'...me': 'e2e-shared-b-w3-17e67b', 'name': 'e2e-shared-b-w3-17e67b', 'namespace': 'ai-tenant-e2e-shared-b-w3-17e67b', ...}) @pytest.fixture(scope="module") def tenant_env(shared_test_tenants): """Adapter fixture with tenant-specific models.""" case_a, case_b = dict(shared_test_tenants[0]), dict(shared_test_tenants[1]) for case in (case_a, case_b): model_name = f"auth-test-model-{case['suffix']}" > provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) test/e2e/tests/test_tenant_auth_isolation.py:99: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:904: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/e2e/tests/test_helper.py:1490: in _create_llmis _apply_cr({ test/e2e/tests/test_helper.py:392: in _apply_cr subprocess.run(["oc", "apply", "-f", "-"], input=json.dumps(cr_dict), capture_output=True, text=True, check=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = '{"apiVersion": "serving.kserve.io/v1alpha1", "kind": "LLMInferenceService", "metadata": {"name": "auth-test-model-480...ttps", "scheme": "HTTPS"}}, "readinessProbe": {"httpGet": {"path": "/ready", "port": "https", "scheme": "HTTPS"}}}]}}}' capture_output = True, timeout = None, check = True popenargs = (['oc', 'apply', '-f', '-'],) kwargs = {'stderr': -1, 'stdin': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'apply', '-f', '-']>, stdout = '' stderr = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "llmi...webhook-server-service.opendatahub.svc:443/validate-serving-kserve-io-v1alpha1-llminferenceservice?timeout=10s": EOF\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'apply', '-f', '-']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessError/workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:133: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:181: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:268: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.