/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f6478756e20> def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid model (will be Active) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Verify it starts as Active cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") log.info(f"Initial phase: {phase}") assert phase == "Active", f"Expected Active initially, got {phase}" # Create API key while Active api_key = _create_api_key( oc_token, name="failed-sub-test", subscription=subscription_name ) # Verify inference works while Active log.info("Testing inference while Active...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) assert r.status_code == 200, f"Expected 200 while Active, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference works with Active subscription") # Manually patch subscription to Failed phase import subprocess import json from datetime import datetime log.info("Manually patching subscription to Failed phase...") patch_data = { "status": { "phase": "Failed", "conditions": [ { "type": "Ready", "status": "False", "reason": "Failed", "message": "Subscription failed", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") } ], "modelRefStatuses": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Model failed" } ] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch to Failed phase: {result.stderr}" # Verify phase is Failed cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") > assert phase == "Failed", f"Expected Failed phase after patch, got {phase}" E AssertionError: Expected Failed phase after patch, got Active E assert 'Active' == 'Failed' E E - Failed E + Active test/e2e/tests/test_subscription.py:2357: AssertionError/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not assertedself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f6478bdca00> def test_aitenant_delete_cleans_up_bootstrap_resources(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) _apply_aitenant(case) _assert_aitenant_bootstrap_resources(case) > _delete_aitenant(case) test/e2e/tests/test_aitenant_lifecycle.py:513: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:250: in _delete_aitenant _delete(AITENANT_KIND, case["aitenant_name"], AITENANT_NAMESPACE, timeout="180s") _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ait-67bdeceb', namespace = 'ai-tenants' def _delete(kind, name, namespace=None, *, timeout="60s"): args = ["delete", kind, name, "--ignore-not-found", f"--timeout={timeout}"] if namespace: args.extend(["-n", namespace]) process_timeout = OC_TIMEOUT + 30 if timeout.endswith("s") and timeout[:-1].isdigit(): process_timeout = max(process_timeout, int(timeout[:-1]) + 30) result = _oc_run(args, timeout=process_timeout) if result.returncode != 0: > raise RuntimeError(f"`oc {' '.join(args)}` failed: {result.stderr.strip() or result.stdout.strip()}") E RuntimeError: `oc delete aitenant e2e-ait-67bdeceb --ignore-not-found --timeout=180s -n ai-tenants` failed: error: timed out waiting for the condition on aitenants/e2e-ait-67bdeceb test/e2e/tests/test_aitenant_lifecycle.py:76: RuntimeError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:353: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f6478be82b0> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase="Active", ) > wait_for_aitenant_cleanup_resources(case) test/e2e/tests/test_multi_tenant_integration.py:111: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:1003: in wait_for_aitenant_cleanup_resources wait_for_json(kind, name, namespace, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'cronjob', name = 'maas-api-key-cleanup-e2e-mt-833acb39' namespace = 'odh-ai-gateway-infra' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: cronjob/maas-api-key-cleanup-e2e-mt-833acb39 in odh-ai-gateway-infra did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:221: AssertionError/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:209: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_Btenant_env = ({'backend_model_name': 'facebook/opt-125m', 'base_url': 'https://e2e-shared-a-e6a009.apps.06d2ae1d-9cc9-4074-b1ec-226...62.prod.konfluxeaas.com/maas-api', 'gateway_name': 'e2e-shared-b-53dad9', 'model_name': 'rate-test-model-53dad9', ...}) @pytest.fixture def tenant_rate_limit_setup(tenant_env): suffix = uuid.uuid4().hex[:6] tenant_a, tenant_b = tenant_env policy_name = f"e2e-rate-iso-auth-{suffix}" sub_a = f"e2e-rate-iso-a-{suffix}" sub_b = f"e2e-rate-iso-b-{suffix}" try: for tenant in tenant_env: apply_maas_auth_policy( policy_name, tenant["namespace"], model_ref=tenant["model_name"], model_namespace=tenant["model_namespace"], ) wait_for_status_phase( "maasauthpolicy", policy_name, tenant["namespace"], expected_phase="Active", ) apply_maas_subscription( sub_a, tenant_a["namespace"], model_ref=tenant_a["model_name"], model_namespace=tenant_a["model_namespace"], token_limit=3, window="1m", ) apply_maas_subscription( sub_b, tenant_b["namespace"], model_ref=tenant_b["model_name"], model_namespace=tenant_b["model_namespace"], token_limit=100, window="1m", ) for name, namespace in ((sub_a, tenant_a["namespace"]), (sub_b, tenant_b["namespace"])): wait_for_status_phase( "maassubscription", name, namespace, expected_phase=("Active", "Degraded"), ) _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=namespace, timeout=120, ) for tenant in (tenant_a, tenant_b): wait_for_status_phase( "maasmodelref", tenant["model_name"], tenant["namespace"], expected_phase="Ready", timeout=180, ) oc_token = _get_cluster_token() key_a_response = create_api_key_at( tenant_a["base_url"], oc_token, f"e2e-rate-a-{suffix}", subscription=sub_a, ) > assert key_a_response.status_code in (200, 201), response_summary(key_a_response) E AssertionError: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_rate_limit_isolation.py:124: AssertionErrortenant_env = ({'backend_model_name': 'facebook/opt-125m', 'base_url': 'https://e2e-shared-a-e6a009.apps.06d2ae1d-9cc9-4074-b1ec-226...62.prod.konfluxeaas.com/maas-api', 'gateway_name': 'e2e-shared-b-53dad9', 'model_name': 'rate-test-model-53dad9', ...}) @pytest.fixture def tenant_rate_limit_setup(tenant_env): suffix = uuid.uuid4().hex[:6] tenant_a, tenant_b = tenant_env policy_name = f"e2e-rate-iso-auth-{suffix}" sub_a = f"e2e-rate-iso-a-{suffix}" sub_b = f"e2e-rate-iso-b-{suffix}" try: for tenant in tenant_env: apply_maas_auth_policy( policy_name, tenant["namespace"], model_ref=tenant["model_name"], model_namespace=tenant["model_namespace"], ) wait_for_status_phase( "maasauthpolicy", policy_name, tenant["namespace"], expected_phase="Active", ) apply_maas_subscription( sub_a, tenant_a["namespace"], model_ref=tenant_a["model_name"], model_namespace=tenant_a["model_namespace"], token_limit=3, window="1m", ) apply_maas_subscription( sub_b, tenant_b["namespace"], model_ref=tenant_b["model_name"], model_namespace=tenant_b["model_namespace"], token_limit=100, window="1m", ) for name, namespace in ((sub_a, tenant_a["namespace"]), (sub_b, tenant_b["namespace"])): wait_for_status_phase( "maassubscription", name, namespace, expected_phase=("Active", "Degraded"), ) _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=namespace, timeout=120, ) for tenant in (tenant_a, tenant_b): wait_for_status_phase( "maasmodelref", tenant["model_name"], tenant["namespace"], expected_phase="Ready", timeout=180, ) oc_token = _get_cluster_token() key_a_response = create_api_key_at( tenant_a["base_url"], oc_token, f"e2e-rate-a-{suffix}", subscription=sub_a, ) > assert key_a_response.status_code in (200, 201), response_summary(key_a_response) E AssertionError: status=401 body= E assert 401 in (200, 201) E + where 401 = <Response [401]>.status_code test/e2e/tests/test_tenant_rate_limit_isolation.py:124: AssertionError