/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not assertedself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f8b6c046b20> def test_aitenant_delete_cleans_up_bootstrap_resources(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) _apply_aitenant(case) _assert_aitenant_bootstrap_resources(case) > _delete_aitenant(case) test/e2e/tests/test_aitenant_lifecycle.py:381: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:235: in _delete_aitenant _delete(AITENANT_KIND, case["aitenant_name"], AITENANT_NAMESPACE) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ait-c9655e49', namespace = 'ai-tenants' def _delete(kind, name, namespace=None, *, timeout="60s"): args = ["delete", kind, name, "--ignore-not-found", f"--timeout={timeout}"] if namespace: args.extend(["-n", namespace]) result = _oc_run(args, timeout=OC_TIMEOUT + 30) if result.returncode != 0: > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc delete aitenant e2e-ait-c9655e49 --ignore-not-found --timeout=60s -n ai-tenants` failed: error: timed out waiting for the condition on aitenants/e2e-ait-c9655e49 test/e2e/tests/test_aitenant_lifecycle.py:67: RuntimeError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f8b6c1695e0> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json("tenant", TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase="Active", ) delete_best_effort(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE) > wait_for_not_found("tenant", TENANT_CR_NAME, case["tenant_ns"], timeout=180) test/e2e/tests/test_multi_tenant_integration.py:110: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'tenant', name = 'default-tenant' namespace = 'ai-tenant-e2e-mt-57198657' def wait_for_not_found( kind: str, name: str, namespace: Optional[str] = None, *, timeout: int = 120, interval: int = 5, ) -> None: deadline = time.time() + timeout while time.time() < deadline: if get_json_or_none(kind, name, namespace) is None: return time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} still exists") E AssertionError: tenant/default-tenant in ai-tenant-e2e-mt-57198657 still exists test/e2e/tests/multitenancy_helpers.py:235: AssertionErrorself = <test_tenant_model_inference.TestTenantModelInference object at 0x7f8b6bd33b50> def test_tenant_deletion_prevents_key_reuse_on_recreation(self): """Tenant deletion revokes keys - orphaned keys don't work if tenant is recreated. Security test (RHOAIENG-72792): When a tenant is deleted and recreated with the same name, old API keys from the deleted tenant should not grant access. Steps: 1. Create AITenant 2. Create API key 3. Delete AITenant 4. Recreate AITenant with same name 5. Verify old API key does not work """ require_aitenant_crd() case = new_named_tenant_case("e2e-reuse") orphaned_key = None try: # Step 1: Create AITenant bootstrap_aitenant_tenant(case) model_name = f"test-model-{case['suffix']}" _create_llmis(model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE) wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180 ) _create_maas_model_ref(model_name, case["tenant_ns"], model_name) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"] ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"] ) wait_for_status_phase("maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180) # Allow AuthPolicy to sync with Authorino before creating API key _wait_reconcile() # Step 2: Create API key gateway_url = _get_tenant_gateway_url(case["gateway_name"]) oc_token = _get_cluster_token() r = requests.post( f"{gateway_url}/maas-api/v1/api-keys", headers={"Authorization": f"Bearer {oc_token}", "Content-Type": "application/json"}, json={"name": "e2e-orphaned-key", "subscription": f"{model_name}-sub"}, timeout=45, verify=TLS_VERIFY, ) > assert r.status_code in (200, 201), f"Failed to create key: {r.status_code}" E AssertionError: Failed to create key: 503 E assert 503 in (200, 201) E + where 503 = <Response [503]>.status_code test/e2e/tests/test_tenant_model_inference.py:333: AssertionError/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:209: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_discovery.py:134: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:210: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:131: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:179: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:266: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:249: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:260: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:274: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:287: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:300: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:332: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:376: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:392: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:402: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:425: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:433: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:438: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:451: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:498: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:534: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:599: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:630: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:656: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:687: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:747: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:771: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:926: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:978: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1014: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1068: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1107: EXTERNAL_OIDC not enabled