_worker_api_keys_context = WorkerTenantContext(worker_id='w2', suffix='0969b1', tenant_name='e2e-worker-w2-0969b1', tenant_namespace='ai-tenant-e...ulator-access', subscription_name='simulator-subscription', premium_subscription_name='premium-simulator-subscription') @pytest.fixture(scope="module") def admin_headers(_worker_api_keys_context): """Use an admin identity in the same tenant as parallel API-key tests.""" context = _worker_api_keys_context if context is None: token = os.environ.get("ADMIN_OC_TOKEN", "") return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} if token else None from worker_tenant_fixtures import xdist_worker_suffix suffix = xdist_worker_suffix() service_account = f"e2e-worker-admin-{suffix}" role_binding = f"{service_account}-binding" _apply_cr( { "apiVersion": "v1", "kind": "ServiceAccount", "metadata": { "name": service_account, "namespace": context.tenant_namespace, }, } ) _apply_cr( { "apiVersion": "rbac.authorization.k8s.io/v1", "kind": "RoleBinding", "metadata": { "name": role_binding, "namespace": context.tenant_namespace, }, "roleRef": { "apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": f"aitenant-{context.tenant_name}-tenant-admin", }, "subjects": [ { "kind": "ServiceAccount", "name": service_account, "namespace": context.tenant_namespace, } ], } ) > token = _create_sa_token(service_account, namespace=context.tenant_namespace, duration="1h") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/tests/test_api_keys.py:186: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-worker-admin-w2', namespace = 'ai-tenant-e2e-worker-w2-0969b1' duration = '1h' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") result = subprocess.run( ["oc", "create", "token", sa_name, "-n", namespace, f"--duration={duration}"], capture_output=True, text=True, ) token = result.stdout.strip() if not token: > raise RuntimeError(f"Could not create token for SA {sa_name}: {result.stderr}") E RuntimeError: Could not create token for SA e2e-worker-admin-w2: error: failed to create token: Post "https://a08c1cf70531f4f9bb3d019389bdf05e-9096eb08df704de8.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/ai-tenant-e2e-worker-w2-0969b1/serviceaccounts/e2e-worker-admin-w2/token": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:524: RuntimeError/workspace/source/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_external_model_routing.py:481: provider withdrawal E2E for #108 is gated; set TENANT_EXTERNAL_MODEL_PARTIAL_WITHDRAWAL_E2E=true to enable (with it set, the #108 contract is expected to hold and a violation fails red)/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/e2e/tests/test_external_model_routing.py:564: provider withdrawal E2E for #106 is gated; set TENANT_EXTERNAL_MODEL_LAST_PROVIDER_WITHDRAWAL_E2E=true to enable (with it set, the #106 contract is expected to hold and a violation fails red)self = <test_models_endpoint.TestModelsEndpoint object at 0x7fc24da4f590> def test_user_token_returns_all_models(self): """ Test: User token automatically returns models from all subscriptions. Creates a user with access to TWO subscriptions containing different models. Queries without X-MaaS-Subscription header and validates: - Returns models from ALL accessible subscriptions - Each model includes subscriptions array showing which subscription(s) provide access - Models appearing in multiple subscriptions have aggregated subscription list """ log.info("Test: User token returns models from all subscriptions") sa_name = "e2e-return-all-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-return-all-sub1" sub2_name = "e2e-return-all-sub2" auth1_name = "e2e-return-all-auth1" auth2_name = "e2e-return-all-auth2" try: # Create SA > sa_token = _create_sa_token(sa_name, namespace=sa_ns) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/tests/test_models_endpoint.py:1275: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-return-all-sa', namespace = 'default', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") result = subprocess.run( ["oc", "create", "token", sa_name, "-n", namespace, f"--duration={duration}"], capture_output=True, text=True, ) token = result.stdout.strip() if not token: > raise RuntimeError(f"Could not create token for SA {sa_name}: {result.stderr}") E RuntimeError: Could not create token for SA e2e-return-all-sa: error: failed to create token: Post "https://a08c1cf70531f4f9bb3d019389bdf05e-9096eb08df704de8.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/default/serviceaccounts/e2e-return-all-sa/token": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:524: RuntimeError/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_networkpolicy.py:119: AIGC praxis NetworkPolicy does not include OTLP 4317 egress (MaaS Go IPP bundle only)/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)/workspace/source/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:251: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:341: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:396: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:434: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:227: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:199: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:307: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:268: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable