/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1054: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f82687f9280> def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid model (will be Active) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Verify it starts as Active cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") log.info(f"Initial phase: {phase}") assert phase == "Active", f"Expected Active initially, got {phase}" # Create API key while Active api_key = _create_api_key( oc_token, name="failed-sub-test", subscription=subscription_name ) # Verify inference works while Active log.info("Testing inference while Active...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) assert r.status_code == 200, f"Expected 200 while Active, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference works with Active subscription") # Manually patch subscription to Failed phase import subprocess import json from datetime import datetime log.info("Manually patching subscription to Failed phase...") patch_data = { "status": { "phase": "Failed", "conditions": [ { "type": "Ready", "status": "False", "reason": "Failed", "message": "Subscription failed", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") } ], "modelRefStatuses": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Model failed" } ] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch to Failed phase: {result.stderr}" # Verify phase is Failed cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") > assert phase == "Failed", f"Expected Failed phase after patch, got {phase}" E AssertionError: Expected Failed phase after patch, got Active E assert 'Active' == 'Failed' E E - Failed E + Active test/e2e/tests/test_subscription.py:2355: AssertionError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7f82682c81c0> def test_tenant_admin_rbac_is_namespace_scoped(self): """1.6: Tenant-admin Role from AITenant bootstrap is scoped to the tenant namespace.""" case = new_discovery_case() sa_name = f"e2e-mt-rbac-{case['suffix']}" role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" other_ns = f"e2e-mt-other-{case['suffix']}" try: apply_gateway_fixture(case["gateway_name"], fixture_label=case["tenant_label_name"]) > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_namespace_discovery.py:255: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:777: in bootstrap_aitenant_tenant apply_aitenant(case) test/e2e/tests/multitenancy_helpers.py:736: in apply_aitenant _apply( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ obj = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'name': 'e2e-mt-d69a5dde', 'namespace': 'ai-tenants'}, 'spec': {'gateway': {'name': 'e2e-mt-d69a5dde'}}} def _apply(obj: dict) -> None: result = _oc_run(["apply", "-f", "-"], input_text=json.dumps(obj)) if result.returncode != 0: > raise RuntimeError(f"`oc apply` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc apply` failed: Error from server (InternalError): error when creating "STDIN": Internal error occurred: failed calling webhook "vaitenant.kb.io": failed to call webhook: Post "https://maas-controller-webhook-service.opendatahub.svc:443/validate-maas-opendatahub-io-v1alpha1-aitenant?timeout=10s": no endpoints available for service "maas-controller-webhook-service" test/e2e/tests/multitenancy_helpers.py:150: RuntimeError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:353: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:212: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_discovery.py:134: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:210: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:131: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:179: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:266: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:249: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:260: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:274: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:287: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:300: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:332: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:376: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:392: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:402: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:425: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:433: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:438: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:451: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:498: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:534: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:599: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:630: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:656: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:687: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:747: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:771: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:926: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:978: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1014: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1068: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1107: EXTERNAL_OIDC not enabled