self = <test_model_identity_conflict.TestModelIdentityConflictDetection object at 0x7f0ab0638bb0> def test_colliding_model_names_flagged_then_resolved(self): suffix = uuid.uuid4().hex[:8] shared_model_name = f"test/e2e-identity-conflict-{suffix}" llmis_a = f"e2e-conflict-a-{suffix}" llmis_b = f"e2e-conflict-b-{suffix}" ref_a, ref_b = llmis_a, llmis_b try: # A single model with no siblings must be reported unique. _create_llmis(llmis_a, MODEL_NAMESPACE, DEFAULT_GATEWAY_NAME, GATEWAY_NAMESPACE, model_name=shared_model_name) _create_maas_model_ref(ref_a, MODEL_NAMESPACE, llmis_a) obj_a = wait_for_status_condition( "maasmodelref", ref_a, MODEL_NAMESPACE, condition_type=CONDITION, expected_status="True", timeout=120, ) assert (obj_a.get("status") or {}).get("resolvedModelAlias"), ( f"expected {ref_a} to have a resolvedModelAlias once the LLMIS exists" ) # A second MaaSModelRef whose LLMIS declares the SAME spec.model.name # introduces a collision — this mirrors the reported production bug. _create_llmis(llmis_b, MODEL_NAMESPACE, DEFAULT_GATEWAY_NAME, GATEWAY_NAMESPACE, model_name=shared_model_name) _create_maas_model_ref(ref_b, MODEL_NAMESPACE, llmis_b) > obj_a = wait_for_status_condition( "maasmodelref", ref_a, MODEL_NAMESPACE, condition_type=CONDITION, expected_status="False", timeout=120, ) test/e2e/tests/test_model_identity_conflict.py:94: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:303: in wait_for_status_condition return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'e2e-conflict-a-ce93b475', namespace = 'llm' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/e2e-conflict-a-ce93b475 in llm did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"e2e-conflict-a-ce93b475","namespace":"llm"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"e2e-conflict-a-ce93b475"}}}\n'}, 'creationTimestamp': '2026-09-08T14:09:19Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'e2e-conflict-a-ce93b475', 'namespace': 'llm', 'resourceVersion': '31204', 'uid': 'a5011add-9252-419c-b5c5-388d61b40186'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'e2e-conflict-a-ce93b475'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:09:20Z', 'message': 'Awaiting governance pairing', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-09-08T14:10:44Z', 'message': 'No active subscription and auth policy pairing found', 'observedGeneration': 1, 'reason': 'NoPairingFound', 'status': 'False', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:12:20Z', 'message': 'Backend is healthy', 'observedGeneration': 1, 'reason': 'RuntimeHealthy', 'status': 'True', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:10:47Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}], 'httpRouteGatewayName': 'maas-default-gateway', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'e2e-conflict-a-ce93b475-kserve-route', 'httpRouteNamespace': 'llm', 'phase': 'Pending', 'resolvedModelAlias': 'publishers/llm/models/test/e2e-identity-conflict-ce93b475', 'resolvedTenantRef': 'models-as-a-service'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_tenant_discovery.py:136: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:212: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:252: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:263: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:277: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:290: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:303: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:335: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:379: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:395: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:405: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:428: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:436: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:441: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:454: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:501: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:537: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:602: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:633: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:659: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:690: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:750: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:786: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:961: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1013: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1049: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1103: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1142: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_subscription.py:1069: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:189: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:355: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:133: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:181: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create models in each tenant namespace for case in (case_a, case_b): model_name = f"test-model-{case['suffix']}" # Track model name early for cleanup case["model_name"] = model_name # Create LLMIS pointing to tenant gateway _create_llmis( model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) # Wait for HTTPRoute to be accepted on the tenant gateway wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180, ) # Create MaaSModelRef _create_maas_model_ref(model_name, case["tenant_ns"], model_name) # Create subscription and auth policy (both needed for Ready status) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], token_limit=10000, ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) # Wait for MaaSModelRef to report Ready > wait_for_status_phase( "maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180, ) test/e2e/tests/test_tenant_model_inference.py:128: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:284: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-cb068d' namespace = 'ai-tenant-e2e-inf-a-cb068d' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-cb068d in ai-tenant-e2e-inf-a-cb068d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-cb068d","namespace":"ai-tenant-e2e-inf-a-cb068d"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-cb068d"}}}\n'}, 'creationTimestamp': '2026-09-08T14:12:14Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-cb068d', 'namespace': 'ai-tenant-e2e-inf-a-cb068d', 'resourceVersion': '31119', 'uid': 'b91d8826-bee8-4133-94b9-52ace8c672c3'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-cb068d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-cb068d', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-cb068d-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-cb068d', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-cb068d/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-cb068d'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create models in each tenant namespace for case in (case_a, case_b): model_name = f"test-model-{case['suffix']}" # Track model name early for cleanup case["model_name"] = model_name # Create LLMIS pointing to tenant gateway _create_llmis( model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) # Wait for HTTPRoute to be accepted on the tenant gateway wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180, ) # Create MaaSModelRef _create_maas_model_ref(model_name, case["tenant_ns"], model_name) # Create subscription and auth policy (both needed for Ready status) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], token_limit=10000, ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) # Wait for MaaSModelRef to report Ready > wait_for_status_phase( "maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180, ) test/e2e/tests/test_tenant_model_inference.py:128: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:284: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-cb068d' namespace = 'ai-tenant-e2e-inf-a-cb068d' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-cb068d in ai-tenant-e2e-inf-a-cb068d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-cb068d","namespace":"ai-tenant-e2e-inf-a-cb068d"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-cb068d"}}}\n'}, 'creationTimestamp': '2026-09-08T14:12:14Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-cb068d', 'namespace': 'ai-tenant-e2e-inf-a-cb068d', 'resourceVersion': '31119', 'uid': 'b91d8826-bee8-4133-94b9-52ace8c672c3'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-cb068d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-cb068d', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-cb068d-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-cb068d', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-cb068d/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-cb068d'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create models in each tenant namespace for case in (case_a, case_b): model_name = f"test-model-{case['suffix']}" # Track model name early for cleanup case["model_name"] = model_name # Create LLMIS pointing to tenant gateway _create_llmis( model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) # Wait for HTTPRoute to be accepted on the tenant gateway wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180, ) # Create MaaSModelRef _create_maas_model_ref(model_name, case["tenant_ns"], model_name) # Create subscription and auth policy (both needed for Ready status) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], token_limit=10000, ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) # Wait for MaaSModelRef to report Ready > wait_for_status_phase( "maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180, ) test/e2e/tests/test_tenant_model_inference.py:128: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:284: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-cb068d' namespace = 'ai-tenant-e2e-inf-a-cb068d' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-cb068d in ai-tenant-e2e-inf-a-cb068d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-cb068d","namespace":"ai-tenant-e2e-inf-a-cb068d"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-cb068d"}}}\n'}, 'creationTimestamp': '2026-09-08T14:12:14Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-cb068d', 'namespace': 'ai-tenant-e2e-inf-a-cb068d', 'resourceVersion': '31119', 'uid': 'b91d8826-bee8-4133-94b9-52ace8c672c3'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-cb068d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-cb068d', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-cb068d-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-cb068d', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-cb068d/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-cb068d'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create models in each tenant namespace for case in (case_a, case_b): model_name = f"test-model-{case['suffix']}" # Track model name early for cleanup case["model_name"] = model_name # Create LLMIS pointing to tenant gateway _create_llmis( model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) # Wait for HTTPRoute to be accepted on the tenant gateway wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180, ) # Create MaaSModelRef _create_maas_model_ref(model_name, case["tenant_ns"], model_name) # Create subscription and auth policy (both needed for Ready status) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], token_limit=10000, ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) # Wait for MaaSModelRef to report Ready > wait_for_status_phase( "maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180, ) test/e2e/tests/test_tenant_model_inference.py:128: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:284: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-cb068d' namespace = 'ai-tenant-e2e-inf-a-cb068d' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-cb068d in ai-tenant-e2e-inf-a-cb068d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-cb068d","namespace":"ai-tenant-e2e-inf-a-cb068d"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-cb068d"}}}\n'}, 'creationTimestamp': '2026-09-08T14:12:14Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-cb068d', 'namespace': 'ai-tenant-e2e-inf-a-cb068d', 'resourceVersion': '31119', 'uid': 'b91d8826-bee8-4133-94b9-52ace8c672c3'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-cb068d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-cb068d', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-cb068d-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-cb068d', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-cb068d/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-cb068d'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create models in each tenant namespace for case in (case_a, case_b): model_name = f"test-model-{case['suffix']}" # Track model name early for cleanup case["model_name"] = model_name # Create LLMIS pointing to tenant gateway _create_llmis( model_name, case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) # Wait for HTTPRoute to be accepted on the tenant gateway wait_for_httproute_accepted( f"{model_name}-kserve-route", case["tenant_ns"], case["gateway_name"], timeout=180, ) # Create MaaSModelRef _create_maas_model_ref(model_name, case["tenant_ns"], model_name) # Create subscription and auth policy (both needed for Ready status) apply_maas_subscription( f"{model_name}-sub", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], token_limit=10000, ) apply_maas_auth_policy( f"{model_name}-auth", case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) # Wait for MaaSModelRef to report Ready > wait_for_status_phase( "maasmodelref", model_name, case["tenant_ns"], expected_phase="Ready", timeout=180, ) test/e2e/tests/test_tenant_model_inference.py:128: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:284: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-cb068d' namespace = 'ai-tenant-e2e-inf-a-cb068d' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-cb068d in ai-tenant-e2e-inf-a-cb068d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-cb068d","namespace":"ai-tenant-e2e-inf-a-cb068d"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-cb068d"}}}\n'}, 'creationTimestamp': '2026-09-08T14:12:14Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-cb068d', 'namespace': 'ai-tenant-e2e-inf-a-cb068d', 'resourceVersion': '31119', 'uid': 'b91d8826-bee8-4133-94b9-52ace8c672c3'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-cb068d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-09-08T14:12:14Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-cb068d', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-cb068d-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-cb068d', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-cb068d/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-cb068d'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError/workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:268: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7ff2ddd24760> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" > provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) test/e2e/tests/test_multi_tenant_integration.py:94: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:912: in provision_tenant_model wait_for_status_condition( test/e2e/tests/multitenancy_helpers.py:303: in wait_for_status_condition return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'e2e-lifecycle-model-ec7c379e' namespace = 'ai-tenant-e2e-mt-ec7c379e' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/e2e-lifecycle-model-ec7c379e in ai-tenant-e2e-mt-ec7c379e did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"e2e-lifecycle-model-ec7c379e","namespace":"ai-tenant-e2e-mt-ec7c379e"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"e2e-lifecycle-model-ec7c379e"}}}\n'}, 'creationTimestamp': '2026-09-08T14:16:48Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'e2e-lifecycle-model-ec7c379e', 'namespace': 'ai-tenant-e2e-mt-ec7c379e', 'resourceVersion': '39096', 'uid': '6181caeb-4a35-4cf6-a70e-d05051a46d9f'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'e2e-lifecycle-model-ec7c379e'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-09-08T14:16:48Z', 'message': 'No active subscription and auth policy pairing found', 'observedGeneration': 1, 'reason': 'NoPairingFound', 'status': 'False', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-09-08T14:16:48Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-09-08T14:16:48Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-09-08T14:16:48Z', 'message': 'Awaiting governance pairing and backend readiness', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-mt-ec7c379e', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'e2e-lifecycle-model-ec7c379e-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-mt-ec7c379e', 'phase': 'Pending', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-mt-ec7c379e/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-mt-ec7c379e'}} test/e2e/tests/multitenancy_helpers.py:229: AssertionError