/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_models_endpoint.TestModelsEndpoint object at 0x7f8324f11dd0> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403. The gateway may reject the revoked key before the request reaches maas-api, in which case the response body is empty. """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_subscription_discovery_ready(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_for_cr_absent("maassubscription", subscription_name, namespace=ns) # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist > assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for API key with deleted subscription, got 500: Internal Server Error. E E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/maas-e2e/test/e2e/tests/test_models_endpoint.py:1643: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7fa1c3544d50> def test_labeled_tenant_namespace_is_discovered(self): """1.1: Controller discovers labeled tenant namespaces and reconciles CRs.""" case = new_discovery_case(use_default_gateway=True) try: apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", case["subscription_name"], case["tenant_ns"], FINALIZER_SUBSCRIPTION) > auth = wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-2b329c15' namespace = 'ai-tenant-e2e-mt-2b329c15' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fa1c33e34c0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-2b329c15 in ai-tenant-e2e-mt-2b329c15 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-2b329c15","namespace":"ai-tenant-e2e-mt-2b329c15"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-30T23:43:06Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-2b329c15', 'namespace': 'ai-tenant-e2e-mt-2b329c15', 'resourceVersion': '60523', 'uid': '6e5360c2-819c-450a-ad1b-daa0fc18fe7a'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:43:06Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 35', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7fe5f4f4f110> def test_dynamic_discovery_after_label_added(self): """1.2 variant: Pre-existing CRs reconcile after namespace gains discovery labels.""" case = new_discovery_case(use_default_gateway=True) try: ensure_namespace(case["tenant_ns"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) time.sleep(10) before = get_json_or_none("maasauthpolicy", case["policy_name"], case["tenant_ns"]) assert before is not None assert FINALIZER_AUTHPOLICY not in ((before.get("metadata") or {}).get("finalizers") or []) apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) > wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-8495d4c2' namespace = 'ai-tenant-e2e-mt-8495d4c2' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fe5f4a06a20> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-8495d4c2 in ai-tenant-e2e-mt-8495d4c2 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-8495d4c2","namespace":"ai-tenant-e2e-mt-8495d4c2"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-30T23:43:16Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-8495d4c2', 'namespace': 'ai-tenant-e2e-mt-8495d4c2', 'resourceVersion': '61592', 'uid': 'aa9f76e9-89a4-4720-9212-dea00ddad27e'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:43:27Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 39', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorself = <test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks object at 0x7f2e3876b510> def test_models_as_a_service_namespace_reconciles(self): ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-default-ns-{suffix}" try: apply_maas_auth_policy(policy_name, ns) wait_for_finalizer("maasauthpolicy", policy_name, ns, FINALIZER_AUTHPOLICY) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:394: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-ns-8080ace6' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f2e3832fec0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-ns-8080ace6 in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-ns-8080ace6","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-30T23:44:55Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-ns-8080ace6', 'namespace': 'models-as-a-service', 'resourceVersion': '65751', 'uid': 'c66cd2bd-af3d-4b46-adcd-a2effe9e9b4f'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:44:55Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 56', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-b-df3b5f', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f694de67380>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-b-df3b5f in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-b-df3b5f","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-b-df3b5f"}}}\n'}, 'creationTimestamp': '2026-10-30T23:52:26Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-b-df3b5f', 'namespace': 'ai-tenants', 'resourceVersion': '83290', 'uid': 'f82d9450-c378-411b-9188-9757630e22b3'}, 'spec': {'gateway': {'name': 'e2e-inf-b-df3b5f'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:52:27Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-b-df3b5f', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-b-df3b5f'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-b-df3b5f', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f694de67380>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-b-df3b5f in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-b-df3b5f","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-b-df3b5f"}}}\n'}, 'creationTimestamp': '2026-10-30T23:52:26Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-b-df3b5f', 'namespace': 'ai-tenants', 'resourceVersion': '83290', 'uid': 'f82d9450-c378-411b-9188-9757630e22b3'}, 'spec': {'gateway': {'name': 'e2e-inf-b-df3b5f'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:52:27Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-b-df3b5f', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-b-df3b5f'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. > provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:171: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1242: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/maas-e2e/test/e2e/tests/test_helper.py:2286: in _create_llmis _apply_cr(llmis) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'name': 'test-model-23f75e', ..., 'router': {'gateway': {'refs': [{'name': 'e2e-inf-b-23f75e', 'namespace': 'openshift-ingress'}]}, 'route': {}}, ...}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for LLMInferenceService ai-tenant-e2e-inf-b-23f75e/test-model-23f75e: Error from server (Timeout): error when applying patch: E {"spec":{"template":{"containers":[{"args":["--port","8000","--model","facebook/opt-125m","--mode","random","--no-mm-encoder-only","--ssl-certfile","/var/run/kserve/tls/tls.crt","--ssl-keyfile","/var/run/kserve/tls/tls.key"],"command":["/app/llm-d-inference-sim"],"image":"ghcr.io/llm-d/llm-d-inference-sim@sha256:c3ba435081a4d032676b218ea34eb3a1c54507da0fade2f6297f9c37894fe0d1","livenessProbe":{"httpGet":{"path":"/health","port":"https","scheme":"HTTPS"}},"name":"main","ports":[{"containerPort":8000,"name":"https","protocol":"TCP"}],"readinessProbe":{"httpGet":{"path":"/ready","port":"https","scheme":"HTTPS"}}}]}}} E to: E Resource: "serving.kserve.io/v1alpha1, Resource=llminferenceservices", GroupVersionKind: "serving.kserve.io/v1alpha1, Kind=LLMInferenceService" E Name: "test-model-23f75e", Namespace: "ai-tenant-e2e-inf-b-23f75e" E for: "STDIN": error when patching "STDIN": Timeout: request did not complete within requested timeout - context deadline exceeded test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. > provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:171: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1242: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/maas-e2e/test/e2e/tests/test_helper.py:2286: in _create_llmis _apply_cr(llmis) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'name': 'test-model-23f75e', ..., 'router': {'gateway': {'refs': [{'name': 'e2e-inf-b-23f75e', 'namespace': 'openshift-ingress'}]}, 'route': {}}, ...}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for LLMInferenceService ai-tenant-e2e-inf-b-23f75e/test-model-23f75e: Error from server (Timeout): error when applying patch: E {"spec":{"template":{"containers":[{"args":["--port","8000","--model","facebook/opt-125m","--mode","random","--no-mm-encoder-only","--ssl-certfile","/var/run/kserve/tls/tls.crt","--ssl-keyfile","/var/run/kserve/tls/tls.key"],"command":["/app/llm-d-inference-sim"],"image":"ghcr.io/llm-d/llm-d-inference-sim@sha256:c3ba435081a4d032676b218ea34eb3a1c54507da0fade2f6297f9c37894fe0d1","livenessProbe":{"httpGet":{"path":"/health","port":"https","scheme":"HTTPS"}},"name":"main","ports":[{"containerPort":8000,"name":"https","protocol":"TCP"}],"readinessProbe":{"httpGet":{"path":"/ready","port":"https","scheme":"HTTPS"}}}]}}} E to: E Resource: "serving.kserve.io/v1alpha1, Resource=llminferenceservices", GroupVersionKind: "serving.kserve.io/v1alpha1, Kind=LLMInferenceService" E Name: "test-model-23f75e", Namespace: "ai-tenant-e2e-inf-b-23f75e" E for: "STDIN": error when patching "STDIN": Timeout: request did not complete within requested timeout - context deadline exceeded test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-82d449-auth' namespace = 'ai-tenant-e2e-inf-a-82d449' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fa1c39ebc40> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-82d449-auth in ai-tenant-e2e-inf-a-82d449 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-82d449-auth","namespace":"ai-tenant-e2e-inf-a-82d449"},"spec":{"modelRefs":[{"name":"test-model-82d449","namespace":"ai-tenant-e2e-inf-a-82d449"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-30T23:52:04Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-82d449-auth', 'namespace': 'ai-tenant-e2e-inf-a-82d449', 'resourceVersion': '88755', 'uid': '96c7c00e-71ed-4384-86d8-a7d97b6078de'}, 'spec': {'modelRefs': [{'name': 'test-model-82d449', 'namespace': 'ai-tenant-e2e-inf-a-82d449'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:52:04Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-82d449-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (cccc33e22a7504ba3e2e8870676dae8eb3f243b9dee4edbbd94255b8dc72e83f) AuthConfig (06617ad46742b394a920c659277f76ef67ea90e3fe7366d977ee3be59f8aa206) AuthConfig (67cddf7f73a05bb2a50b2f53ab6e164e8c924653bfe3a7bed64b7032c09dd7aa) AuthConfig (a69e0f2695aa52f8617d9d8e07c1632a3c411c0f6216a559a8cad51c070bb2c8) AuthConfig (6546b13ad44aa5e36d21ab334716f620c7c4f2c225f8854c99e3fff078f62729)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-82d449-auth' namespace = 'ai-tenant-e2e-inf-a-82d449' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fa1c39ebc40> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-82d449-auth in ai-tenant-e2e-inf-a-82d449 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-82d449-auth","namespace":"ai-tenant-e2e-inf-a-82d449"},"spec":{"modelRefs":[{"name":"test-model-82d449","namespace":"ai-tenant-e2e-inf-a-82d449"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-30T23:52:04Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-82d449-auth', 'namespace': 'ai-tenant-e2e-inf-a-82d449', 'resourceVersion': '88755', 'uid': '96c7c00e-71ed-4384-86d8-a7d97b6078de'}, 'spec': {'modelRefs': [{'name': 'test-model-82d449', 'namespace': 'ai-tenant-e2e-inf-a-82d449'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:52:04Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-82d449-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (cccc33e22a7504ba3e2e8870676dae8eb3f243b9dee4edbbd94255b8dc72e83f) AuthConfig (06617ad46742b394a920c659277f76ef67ea90e3fe7366d977ee3be59f8aa206) AuthConfig (67cddf7f73a05bb2a50b2f53ab6e164e8c924653bfe3a7bed64b7032c09dd7aa) AuthConfig (a69e0f2695aa52f8617d9d8e07c1632a3c411c0f6216a559a8cad51c070bb2c8) AuthConfig (6546b13ad44aa5e36d21ab334716f620c7c4f2c225f8854c99e3fff078f62729)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. > provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:171: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1242: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/maas-e2e/test/e2e/tests/test_helper.py:2286: in _create_llmis _apply_cr(llmis) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'name': 'test-model-f0817c', ..., 'router': {'gateway': {'refs': [{'name': 'e2e-inf-b-f0817c', 'namespace': 'openshift-ingress'}]}, 'route': {}}, ...}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for LLMInferenceService ai-tenant-e2e-inf-b-f0817c/test-model-f0817c: Error from server (Timeout): error when applying patch: E {"spec":{"template":{"containers":[{"args":["--port","8000","--model","facebook/opt-125m","--mode","random","--no-mm-encoder-only","--ssl-certfile","/var/run/kserve/tls/tls.crt","--ssl-keyfile","/var/run/kserve/tls/tls.key"],"command":["/app/llm-d-inference-sim"],"image":"ghcr.io/llm-d/llm-d-inference-sim@sha256:c3ba435081a4d032676b218ea34eb3a1c54507da0fade2f6297f9c37894fe0d1","livenessProbe":{"httpGet":{"path":"/health","port":"https","scheme":"HTTPS"}},"name":"main","ports":[{"containerPort":8000,"name":"https","protocol":"TCP"}],"readinessProbe":{"httpGet":{"path":"/ready","port":"https","scheme":"HTTPS"}}}]}}} E to: E Resource: "serving.kserve.io/v1alpha1, Resource=llminferenceservices", GroupVersionKind: "serving.kserve.io/v1alpha1, Kind=LLMInferenceService" E Name: "test-model-f0817c", Namespace: "ai-tenant-e2e-inf-b-f0817c" E for: "STDIN": error when patching "STDIN": Timeout: request did not complete within requested timeout - context deadline exceeded test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. > provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:171: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1242: in provision_tenant_model _create_llmis(model_name, tenant_namespace, gateway_name, GATEWAY_NAMESPACE) test/maas-e2e/test/e2e/tests/test_helper.py:2286: in _create_llmis _apply_cr(llmis) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'serving.kserve.io/v1alpha1', 'kind': 'LLMInferenceService', 'metadata': {'name': 'test-model-f0817c', ..., 'router': {'gateway': {'refs': [{'name': 'e2e-inf-b-f0817c', 'namespace': 'openshift-ingress'}]}, 'route': {}}, ...}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for LLMInferenceService ai-tenant-e2e-inf-b-f0817c/test-model-f0817c: Error from server (Timeout): error when applying patch: E {"spec":{"template":{"containers":[{"args":["--port","8000","--model","facebook/opt-125m","--mode","random","--no-mm-encoder-only","--ssl-certfile","/var/run/kserve/tls/tls.crt","--ssl-keyfile","/var/run/kserve/tls/tls.key"],"command":["/app/llm-d-inference-sim"],"image":"ghcr.io/llm-d/llm-d-inference-sim@sha256:c3ba435081a4d032676b218ea34eb3a1c54507da0fade2f6297f9c37894fe0d1","livenessProbe":{"httpGet":{"path":"/health","port":"https","scheme":"HTTPS"}},"name":"main","ports":[{"containerPort":8000,"name":"https","protocol":"TCP"}],"readinessProbe":{"httpGet":{"path":"/ready","port":"https","scheme":"HTTPS"}}}]}}} E to: E Resource: "serving.kserve.io/v1alpha1, Resource=llminferenceservices", GroupVersionKind: "serving.kserve.io/v1alpha1, Kind=LLMInferenceService" E Name: "test-model-f0817c", Namespace: "ai-tenant-e2e-inf-b-f0817c" E for: "STDIN": error when patching "STDIN": Timeout: request did not complete within requested timeout - context deadline exceeded test/maas-e2e/test/e2e/tests/test_helper.py:767: RuntimeError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1327: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-1bc837' namespace = 'ai-tenant-e2e-inf-b-1bc837' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fe5f50c3100> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-1bc837 in ai-tenant-e2e-inf-b-1bc837 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-1bc837","namespace":"ai-tenant-e2e-inf-b-1bc837"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-1bc837"}}}\n'}, 'creationTimestamp': '2026-10-30T23:52:00Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-1bc837', 'namespace': 'ai-tenant-e2e-inf-b-1bc837', 'resourceVersion': '88239', 'uid': '41451c39-3fc1-4f41-899a-938f37bf3030'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-1bc837'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:53:33Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-10-30T23:52:00Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-b-1bc837', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-1bc837-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-b-1bc837', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-b-1bc837/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-b-1bc837'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1327: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-1bc837' namespace = 'ai-tenant-e2e-inf-b-1bc837' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fe5f50c3100> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-1bc837 in ai-tenant-e2e-inf-b-1bc837 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-1bc837","namespace":"ai-tenant-e2e-inf-b-1bc837"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-1bc837"}}}\n'}, 'creationTimestamp': '2026-10-30T23:52:00Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-1bc837', 'namespace': 'ai-tenant-e2e-inf-b-1bc837', 'resourceVersion': '88239', 'uid': '41451c39-3fc1-4f41-899a-938f37bf3030'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-1bc837'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:53:33Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-10-30T23:52:00Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-b-1bc837', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-1bc837-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-b-1bc837', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-b-1bc837/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-b-1bc837'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1327: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-a63288' namespace = 'ai-tenant-e2e-inf-a-a63288' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f2e3897c7c0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-a63288 in ai-tenant-e2e-inf-a-a63288 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-a63288","namespace":"ai-tenant-e2e-inf-a-a63288"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-a63288"}}}\n'}, 'creationTimestamp': '2026-10-30T23:51:49Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-a63288', 'namespace': 'ai-tenant-e2e-inf-a-a63288', 'resourceVersion': '88219', 'uid': '6d1e565f-4598-4777-a472-5bead97acbd5'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-a63288'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:53:32Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-10-30T23:51:49Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-a63288', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-a63288-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-a63288', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-a63288/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-a63288'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1327: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasmodelref', name = 'test-model-a63288' namespace = 'ai-tenant-e2e-inf-a-a63288' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f2e3897c7c0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasmodelref/test-model-a63288 in ai-tenant-e2e-inf-a-a63288 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSModelRef', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSModelRef","metadata":{"annotations":{},"name":"test-model-a63288","namespace":"ai-tenant-e2e-inf-a-a63288"},"spec":{"modelRef":{"kind":"LLMInferenceService","name":"test-model-a63288"}}}\n'}, 'creationTimestamp': '2026-10-30T23:51:49Z', 'finalizers': ['maas.opendatahub.io/model-cleanup'], 'generation': 1, 'name': 'test-model-a63288', 'namespace': 'ai-tenant-e2e-inf-a-a63288', 'resourceVersion': '88219', 'uid': '6d1e565f-4598-4777-a472-5bead97acbd5'}, 'spec': {'modelRef': {'kind': 'LLMInferenceService', 'name': 'test-model-a63288'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T23:53:32Z', 'message': 'Active governance pairing found', 'observedGeneration': 1, 'reason': 'GovernancePaired', 'status': 'True', 'type': 'GovernanceAttached'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Backend is not ready', 'observedGeneration': 1, 'reason': 'RuntimeHealthFailure', 'status': 'False', 'type': 'RuntimeReady'}, {'lastTransitionTime': '2026-10-30T23:51:49Z', 'message': 'No other MaaSModelRef in this namespace resolves to the same model identity', 'observedGeneration': 1, 'reason': 'UniqueIdentity', 'status': 'True', 'type': 'ModelIdentityUnique'}, {'lastTransitionTime': '2026-10-30T23:54:23Z', 'message': 'Governed but backend is not ready', 'observedGeneration': 1, 'reason': 'BackendNotReady', 'status': 'False', 'type': 'Ready'}], 'httpRouteGatewayName': 'e2e-inf-a-a63288', 'httpRouteGatewayNamespace': 'openshift-ingress', 'httpRouteName': 'test-model-a63288-kserve-route', 'httpRouteNamespace': 'ai-tenant-e2e-inf-a-a63288', 'phase': 'Unhealthy', 'resolvedModelAlias': 'publishers/ai-tenant-e2e-inf-a-a63288/models/facebook/opt-125m', 'resolvedTenantRef': 'e2e-inf-a-a63288'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError