/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_negative_security.TestWebhookValidation object at 0x7fe3c7ec81f0> def test_subscription_rejected_in_unlabeled_namespace(self): """MaaSSubscription create is rejected in namespace without MaasTenantConfig CR. Webhooks require namespaces to have a MaasTenantConfig CR to contain tenant resources. """ test_ns = f"e2e-webhook-test-{uuid.uuid4().hex[:6]}" try: # Create namespace without MaasTenantConfig CR result = subprocess.run( ["oc", "create", "namespace", test_ns], capture_output=True, text=True, timeout=30 ) assert result.returncode == 0, f"Failed to create namespace: {result.stderr}" # Try to create MaaSSubscription (should be rejected by webhook) result = subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSSubscription", "metadata": {"name": "test-sub", "namespace": test_ns}, "spec": { "owner": {"groups": [{"name": "system:authenticated"}]}, "modelRefs": [{ "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "tokenRateLimits": [{"limit": 100, "window": "1m"}] }], }, }), capture_output=True, text=True, timeout=30 ) # Verify webhook rejection assert result.returncode != 0, "Expected webhook to reject subscription in namespace without MaasTenantConfig CR" assert "admission webhook" in result.stderr.lower(), \ f"Expected webhook rejection, got: {result.stderr}" assert "not enabled for MaaS tenant resources" in result.stderr, \ f"Expected helpful error message, got: {result.stderr}" assert "Create a MaasTenantConfig CR" in result.stderr, \ f"Expected error to mention creating MaasTenantConfig CR, got: {result.stderr}" log.info("✅ Webhook correctly rejected MaaSSubscription in namespace without MaasTenantConfig CR") log.info(f"Error message: {result.stderr}") finally: # Clean up namespace > subprocess.run( ["oc", "delete", "namespace", test_ns, "--ignore-not-found"], capture_output=True, text=True, timeout=30 ) test/maas-e2e/test/e2e/tests/test_negative_security.py:672: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib64/python3.9/subprocess.py:507: in run stdout, stderr = process.communicate(input, timeout=timeout) /usr/lib64/python3.9/subprocess.py:1134: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.9/subprocess.py:1996: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Popen: returncode: -9 args: ['oc', 'delete', 'namespace', 'e2e-webhook-test...> endtime = 114575.603695584, orig_timeout = 30 stdout_seq = [b'namespace "e2e-webhook-test-2dc7de" deleted\n'], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['oc', 'delete', 'namespace', 'e2e-webhook-test-2dc7de', '--ignore-not-found']' timed out after 30 seconds /usr/lib64/python3.9/subprocess.py:1178: TimeoutExpiredself = <test_negative_security.TestWebhookValidation object at 0x7fe3c7ed3b80> def test_authpolicy_rejected_in_unlabeled_namespace(self): """MaaSAuthPolicy create is rejected in namespace without MaasTenantConfig CR.""" test_ns = f"e2e-webhook-test-{uuid.uuid4().hex[:6]}" try: # Create namespace without MaasTenantConfig CR result = subprocess.run( ["oc", "create", "namespace", test_ns], capture_output=True, text=True, timeout=30 ) assert result.returncode == 0, f"Failed to create namespace: {result.stderr}" # Try to create MaaSAuthPolicy (should be rejected by webhook) result = subprocess.run( ["oc", "apply", "-f", "-"], input=json.dumps({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSAuthPolicy", "metadata": {"name": "test-policy", "namespace": test_ns}, "spec": { "modelRefs": [{"name": MODEL_REF, "namespace": MODEL_NAMESPACE}], "subjects": {"groups": [{"name": "system:authenticated"}]}, }, }), capture_output=True, text=True, timeout=30 ) # Verify webhook rejection assert result.returncode != 0, "Expected webhook to reject auth policy in namespace without MaasTenantConfig CR" > assert "admission webhook" in result.stderr.lower(), \ f"Expected webhook rejection, got: {result.stderr}" E AssertionError: Expected webhook rejection, got: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF E E assert 'admission webhook' in 'error from server (internalerror): error when creating "stdin": internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": eof\n' E + where 'error from server (internalerror): error when creating "stdin": internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": eof\n' = <built-in method lower of str object at 0x7fe3c7ad8df0>() E + where <built-in method lower of str object at 0x7fe3c7ad8df0> = 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF\n'.lower E + where 'Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF\n' = CompletedProcess(args=['oc', 'apply', '-f', '-'], returncode=1, stdout='', stderr='Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vmaasauthpolicy.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-maasauthpolicy?timeout=10s": EOF\n').stderr test/maas-e2e/test/e2e/tests/test_negative_security.py:706: AssertionErrorself = <test_negative_security.TestInternalEndpointIsolation object at 0x7fe3c7ed3190> path = '/internal/v1/subscriptions/select' body = {'groups': ['system:authenticated'], 'username': 'test'} @pytest.mark.parametrize( "path,body", [ ( "/internal/v1/subscriptions/select", {"groups": ["system:authenticated"], "username": "test"}, ), ("/internal/v1/api-keys/cleanup", {}), ("/internal/v1/api-keys/validate", {"key": "sk-oai-test-fake-key"}), ], ids=["subscriptions-select", "api-keys-cleanup", "api-keys-validate"], ) def test_internal_endpoint_not_routable(self, path, body): """POST /maas-api/internal/* through gateway must not reach the handler. Sends an authenticated request to each internal endpoint via the gateway. With the HTTPRoute scoped to /maas-api/v1, these paths have no matching route and the gateway returns a non-success status (typically 404). The response must not contain handler output. """ > _wait_for_gateway_auth_enforced() test/maas-e2e/test/e2e/tests/test_negative_security.py:757: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1222: in _wait_for_gateway_auth_enforced cr = _get_cr("authpolicy", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'authpolicy', name = 'maas-gateway-auth', namespace = 'openshift-ingress' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get authpolicy/maas-gateway-auth in namespace 'openshift-ingress': Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/test_helper.py:688: RuntimeErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6372525af0> def test_aitenant_create_bootstrap_resources(self): case = _new_aitenant_case() try: > _apply_gateway_fixture(case) test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:420: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:183: in _apply_gateway_fixture apply_https_gateway_fixture( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:548: in apply_gateway_fixture apply_gateway_access_label(INFRA_NAMESPACE, gateway_name) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:468: in apply_gateway_access_label ensure_namespace(namespace, labels={gateway_access_label_key(gateway_name): "true"}) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'odh-ai-gateway-infra' def ensure_namespace(name: str, *, labels: Optional[dict[str, str]] = None) -> None: result = _oc_run(["create", "namespace", name]) if result.returncode != 0 and "AlreadyExists" not in (result.stderr or "") and "already exists" not in (result.stderr or "").lower(): > raise RuntimeError(f"failed to create namespace {name}: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: failed to create namespace odh-ai-gateway-infra: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:481: RuntimeErrorself = <test_negative_security.TestInternalEndpointIsolation object at 0x7fe3c7ed3f70> path = '/internal/v1/api-keys/cleanup', body = {} @pytest.mark.parametrize( "path,body", [ ( "/internal/v1/subscriptions/select", {"groups": ["system:authenticated"], "username": "test"}, ), ("/internal/v1/api-keys/cleanup", {}), ("/internal/v1/api-keys/validate", {"key": "sk-oai-test-fake-key"}), ], ids=["subscriptions-select", "api-keys-cleanup", "api-keys-validate"], ) def test_internal_endpoint_not_routable(self, path, body): """POST /maas-api/internal/* through gateway must not reach the handler. Sends an authenticated request to each internal endpoint via the gateway. With the HTTPRoute scoped to /maas-api/v1, these paths have no matching route and the gateway returns a non-success status (typically 404). The response must not contain handler output. """ > _wait_for_gateway_auth_enforced() test/maas-e2e/test/e2e/tests/test_negative_security.py:757: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1222: in _wait_for_gateway_auth_enforced cr = _get_cr("authpolicy", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'authpolicy', name = 'maas-gateway-auth', namespace = 'openshift-ingress' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get authpolicy/maas-gateway-auth in namespace 'openshift-ingress': Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/test_helper.py:688: RuntimeErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6372525ee0> def test_aitenant_migrates_and_removes_legacy_tenant(self): > if not _crd_exists(LEGACY_TENANT_CRD): test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:427: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ crd = 'tenants.maas.opendatahub.io' def _crd_exists(crd): result = _oc_run(["get", "crd", crd]) if result.returncode == 0: return True if _oc_output_not_found(result): return False > pytest.fail(f"`oc get crd {crd}` failed: {result.stderr.strip() or result.stdout.strip()}") E Failed: `oc get crd tenants.maas.opendatahub.io` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:148: Failedself = <test_negative_security.TestInternalEndpointIsolation object at 0x7fe3c7edb190> path = '/internal/v1/api-keys/validate', body = {'key': 'sk-oai-test-fake-key'} @pytest.mark.parametrize( "path,body", [ ( "/internal/v1/subscriptions/select", {"groups": ["system:authenticated"], "username": "test"}, ), ("/internal/v1/api-keys/cleanup", {}), ("/internal/v1/api-keys/validate", {"key": "sk-oai-test-fake-key"}), ], ids=["subscriptions-select", "api-keys-cleanup", "api-keys-validate"], ) def test_internal_endpoint_not_routable(self, path, body): """POST /maas-api/internal/* through gateway must not reach the handler. Sends an authenticated request to each internal endpoint via the gateway. With the HTTPRoute scoped to /maas-api/v1, these paths have no matching route and the gateway returns a non-success status (typically 404). The response must not contain handler output. """ > _wait_for_gateway_auth_enforced() test/maas-e2e/test/e2e/tests/test_negative_security.py:757: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1222: in _wait_for_gateway_auth_enforced cr = _get_cr("authpolicy", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'authpolicy', name = 'maas-gateway-auth', namespace = 'openshift-ingress' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get authpolicy/maas-gateway-auth in namespace 'openshift-ingress': Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/test_helper.py:688: RuntimeErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6372525040> def test_aitenant_delete_cleans_maas_resources_and_preserves_user_objects(self): case = _new_aitenant_case() try: > _apply_gateway_fixture(case) test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:540: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:183: in _apply_gateway_fixture apply_https_gateway_fixture( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:548: in apply_gateway_fixture apply_gateway_access_label(INFRA_NAMESPACE, gateway_name) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:468: in apply_gateway_access_label ensure_namespace(namespace, labels={gateway_access_label_key(gateway_name): "true"}) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'odh-ai-gateway-infra' def ensure_namespace(name: str, *, labels: Optional[dict[str, str]] = None) -> None: result = _oc_run(["create", "namespace", name]) if result.returncode != 0 and "AlreadyExists" not in (result.stderr or "") and "already exists" not in (result.stderr or "").lower(): > raise RuntimeError(f"failed to create namespace {name}: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: failed to create namespace odh-ai-gateway-infra: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:481: RuntimeErrorrequest = <SubRequest '_worker_api_keys_context' for <Function test_create_api_key>> @pytest.fixture(scope="module", autouse=True) def _worker_api_keys_context(request): """Bind parallel API-key tests to worker state; leave the serial pass alone.""" from worker_tenant_fixtures import activate_worker_tenant, serial_only_selection if serial_only_selection(request): yield None return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_api_keys.py:84: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w0-6ddb93', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w0-6ddb93 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_models_context' for <Function test_explicit_subscription_header>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_models_endpoint.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w3-74bb52', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w3-74bb52 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_api_keys_context' for <Function test_create_api_key>> @pytest.fixture(scope="module", autouse=True) def _worker_api_keys_context(request): """Bind parallel API-key tests to worker state; leave the serial pass alone.""" from worker_tenant_fixtures import activate_worker_tenant, serial_only_selection if serial_only_selection(request): yield None return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_api_keys.py:84: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w0-6ddb93', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w0-6ddb93 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_models_context' for <Function test_explicit_subscription_header>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_models_endpoint.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w3-74bb52', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w3-74bb52 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_api_keys_context' for <Function test_create_api_key>> @pytest.fixture(scope="module", autouse=True) def _worker_api_keys_context(request): """Bind parallel API-key tests to worker state; leave the serial pass alone.""" from worker_tenant_fixtures import activate_worker_tenant, serial_only_selection if serial_only_selection(request): yield None return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_api_keys.py:84: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w0-6ddb93', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w0-6ddb93 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_models_context' for <Function test_explicit_subscription_header>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_models_endpoint.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w3-74bb52', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w3-74bb52 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_api_keys_context' for <Function test_create_api_key>> @pytest.fixture(scope="module", autouse=True) def _worker_api_keys_context(request): """Bind parallel API-key tests to worker state; leave the serial pass alone.""" from worker_tenant_fixtures import activate_worker_tenant, serial_only_selection if serial_only_selection(request): yield None return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_api_keys.py:84: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w0-6ddb93', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w0-6ddb93 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_models_context' for <Function test_explicit_subscription_header>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_models_endpoint.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w3-74bb52', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w3-74bb52 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_api_keys_context' for <Function test_create_api_key>> @pytest.fixture(scope="module", autouse=True) def _worker_api_keys_context(request): """Bind parallel API-key tests to worker state; leave the serial pass alone.""" from worker_tenant_fixtures import activate_worker_tenant, serial_only_selection if serial_only_selection(request): yield None return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_api_keys.py:84: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w0-6ddb93', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w0-6ddb93 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorrequest = <SubRequest '_worker_models_context' for <Function test_explicit_subscription_header>> @pytest.fixture(scope="module", autouse=True) def _worker_models_context(request): """Route parallel model tests through the explicit worker context. The serial pass intentionally retains the default deployment. """ from worker_tenant_fixtures import ( activate_worker_tenant, ensure_worker_models, serial_only_selection, ) if serial_only_selection(request): yield return > context = request.getfixturevalue("worker_tenant_context") test/maas-e2e/test/e2e/tests/test_models_endpoint.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:549: in getfixturevalue fixturedef = self._get_active_fixturedef(argname) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:640: in _get_active_fixturedef fixturedef.execute(request=subrequest) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1128: in execute result = ihook.pytest_fixture_setup(fixturedef=self, request=request) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_hooks.py:512: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/setuponly.py:36: in pytest_fixture_setup return (yield) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:1196: in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) test/e2e/.venv/lib64/python3.9/site-packages/_pytest/fixtures.py:923: in call_fixture_func fixture_result = next(generator) test/maas-e2e/test/e2e/tests/conftest.py:37: in worker_tenant_context case = bootstrap_worker_tenant(case) test/maas-e2e/test/e2e/tests/worker_tenant_fixtures.py:235: in bootstrap_worker_tenant bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-worker-w3-74bb52', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-worker-w3-74bb52 -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6372525b50> def test_aitenant_derives_non_default_tenant_namespace(self): """RHOAIENG-66836: non-default AITenant must not use models-as-a-service tenant namespace.""" suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-derive-{suffix}" reserved_ns = _ns() expected_ns = f"ai-tenant-{aitenant_name}" gateway_name = aitenant_name try: > _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:601: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:183: in _apply_gateway_fixture apply_https_gateway_fixture( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:548: in apply_gateway_fixture apply_gateway_access_label(INFRA_NAMESPACE, gateway_name) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:468: in apply_gateway_access_label ensure_namespace(namespace, labels={gateway_access_label_key(gateway_name): "true"}) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'odh-ai-gateway-infra' def ensure_namespace(name: str, *, labels: Optional[dict[str, str]] = None) -> None: result = _oc_run(["create", "namespace", name]) if result.returncode != 0 and "AlreadyExists" not in (result.stderr or "") and "already exists" not in (result.stderr or "").lower(): > raise RuntimeError(f"failed to create namespace {name}: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: failed to create namespace odh-ai-gateway-infra: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:481: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:85: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-26ffff', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-inf-a-26ffff -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:85: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-26ffff', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-inf-a-26ffff -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:85: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-26ffff', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-inf-a-26ffff -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:85: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-26ffff', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-inf-a-26ffff -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:85: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:838: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:223: in wait_for_json obj = get_json_or_none(kind, name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-26ffff', namespace = 'ai-tenants' def get_json_or_none(kind: str, name: str, namespace: Optional[str] = None) -> Optional[dict]: args = ["get", kind, name, "-o", "json"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args) if result.returncode == 0: return json.loads(result.stdout) if _oc_output_not_found(result): return None > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc get aitenant e2e-inf-a-26ffff -o json -n ai-tenants` failed: Unable to connect to the server: net/http: TLS handshake timeout test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:194: RuntimeError@pytest.fixture(scope="module", autouse=True) def _require_multitenancy_prerequisites(): > require_tenant_namespace_discovery() test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:73: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:407: in require_tenant_namespace_discovery if not controller_has_tenant_namespace_discovery(): test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:365: in controller_has_tenant_namespace_discovery result = _oc_run(["get", "deployment", "maas-controller", "-n", DEPLOYMENT_NAMESPACE, "-o", "json"]) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:86: in _oc_run return subprocess.run( /usr/lib64/python3.9/subprocess.py:507: in run stdout, stderr = process.communicate(input, timeout=timeout) /usr/lib64/python3.9/subprocess.py:1134: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.9/subprocess.py:1996: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Popen: returncode: -9 args: ['/usr/local/bin/oc', 'get', 'deployment', 'maa...> endtime = 114956.716666878, orig_timeout = 60, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['/usr/local/bin/oc', 'get', 'deployment', 'maas-controller', '-n', 'opendatahub', '-o', 'json']' timed out after 60 seconds /usr/lib64/python3.9/subprocess.py:1178: TimeoutExpired