/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_subscription_list_endpoints.TestListSubscriptions object at 0x7f3f6657d010> endpoint = '/v1/subscriptions' @pytest.mark.parametrize( "endpoint", [ "/v1/subscriptions", f"/v1/model/{MODEL_REF}/subscriptions", ], ids=["all-subscriptions", "subscriptions-for-model"], ) def test_unauthenticated_returns_401(self, endpoint): """Both subscription-list endpoints require authentication.""" url = f"{_maas_api_url()}{endpoint}" r = requests.get(url, timeout=TIMEOUT, verify=TLS_VERIFY) > assert r.status_code == 401, f"Expected 401, got {r.status_code}: {r.text}" E AssertionError: Expected 401, got 500: Internal Server Error. E E assert 500 == 401 E + where 500 = <Response [500]>.status_code test/maas-e2e/test/e2e/tests/test_subscription_list_endpoints.py:282: AssertionErrorself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7f868f5a8210> def test_labeled_tenant_namespace_is_discovered(self): """1.1: Controller discovers labeled tenant namespaces and reconciles CRs.""" case = new_discovery_case(use_default_gateway=True) try: apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", case["subscription_name"], case["tenant_ns"], FINALIZER_SUBSCRIPTION) > auth = wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-3a1b6e60' namespace = 'ai-tenant-e2e-mt-3a1b6e60' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f868f52dbc0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-3a1b6e60 in ai-tenant-e2e-mt-3a1b6e60 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-3a1b6e60","namespace":"ai-tenant-e2e-mt-3a1b6e60"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-01T12:45:02Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-3a1b6e60', 'namespace': 'ai-tenant-e2e-mt-3a1b6e60', 'resourceVersion': '48685', 'uid': 'c49f675b-5506-4eb3-8244-e3cab268dbf5'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:45:03Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 5, current generation is 29', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f46d3f7ab10> def test_aitenant_derives_non_default_tenant_namespace(self): """RHOAIENG-66836: non-default AITenant must not use models-as-a-service tenant namespace.""" suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-derive-{suffix}" reserved_ns = _ns() expected_ns = f"ai-tenant-{aitenant_name}" gateway_name = aitenant_name try: _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "AITenant", "metadata": {"name": aitenant_name, "namespace": AITENANT_NAMESPACE}, "spec": {}, } ) > aitenant = _wait_for_json( AITENANT_KIND, aitenant_name, AITENANT_NAMESPACE, predicate=_aitenant_ready, timeout=120, ) test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:689: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-derive-b2ccbb55', namespace = 'ai-tenants' predicate = <function _aitenant_ready at 0x7f46d850a0c0>, timeout = 120 interval = 5 def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: aitenant/e2e-derive-b2ccbb55 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-derive-b2ccbb55","namespace":"ai-tenants"},"spec":{}}\n'}, 'creationTimestamp': '2026-11-01T12:45:11Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-derive-b2ccbb55', 'namespace': 'ai-tenants', 'resourceVersion': '44086', 'uid': '82f2c11c-be27-4b93-b401-b5f4be39890b'}, 'spec': {}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:45:14Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-derive-b2ccbb55', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-derive-b2ccbb55'}} test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:170: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks object at 0x7f2deb21de50> def test_models_as_a_service_namespace_reconciles(self): ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-default-ns-{suffix}" try: apply_maas_auth_policy(policy_name, ns) wait_for_finalizer("maasauthpolicy", policy_name, ns, FINALIZER_AUTHPOLICY) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:394: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-ns-bf3ef34c' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f2deae1f380> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-ns-bf3ef34c in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-ns-bf3ef34c","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-01T12:46:04Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-ns-bf3ef34c', 'namespace': 'models-as-a-service', 'resourceVersion': '50501', 'uid': '251451ee-a6c7-44d7-bc55-6ec8516072af'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:46:04Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (eaeb360b41b3f9e0bd6b247d4dae3eef1ffc3355d6e9408df3f8f71bc7f033b5) AuthConfig (7c924aa2b37ea38067b4e69052d541ea2e7c86fbdaa79b19f9ea09e87adbe4ff) AuthConfig (5c92635c142b3184e91a78fe113800dddc971eed3a8849837afdf52df706aa5e) AuthConfig (05112428ba8242e3d6eb64227c365ab9f0e9e873d13761ea2e2a4738ccab4a0c) AuthConfig (e5f81c7f7812462ebef000c68dc043013159f22201d2eeecf05c220e809f96ed) AuthConfig (a82769ff87473da04c27497e160e9b4b1a5dd93578e0cf2342be8b2f06845269) AuthConfig (a91faccafad197e2a4065ce3d12a052ac484d2e8fb2fc9265ee48f02b28e6dce) AuthConfig (2f53b030326b8e1f5908c601aecd3fffcb6d08a8337d0bdc53ee1fc699a4cd71) AuthConfig (a7740f6e77d024a437502c023fb1d8939cb3cc2b9ed7f4073af7d779e19459eb) AuthConfig (362c4db530a1f2b250b39b0b7388869d8386a8e10d881cefc4c02ca9e97590ec) AuthConfig (9e75e9fd02f51c2f043b145c5d6b82e95f1447b388ee529262aea27a6a2774ef) AuthConfig (0020c64019ffd1a0bdcc75e6441e6a30e6d8bb08f463f67bd2408a1e82b95b2a) AuthConfig (d752a44ad8df3a50b5a1147110eb89039ed764e1fed197b1f77e0935c424a90b) AuthConfig (0e89fc4b75d3a5dd95fca2c4d8ae33c7826f05a8016c44eb39e28c3c1860fcfb) AuthConfig (100022a5906a36e4db6527195deaa1f3eba3fada253c82ce0002a171c3d2c1f0) AuthConfig (134a7a790ba665547ca14f80f917549f66ee5063fc41096471c0778399db6dd5) AuthConfig (4f800155f293f82eeaf5fd6a9553c0a1bee90f47a295bba8d7a04b3f11faea33) AuthConfig (cad6ab5a81a65fd4cb03a0fefdab5de82aa18c30a5b95078b1d05214b0606f40) AuthConfig (98ec9d74d07c88c1e6923ed1cbafd4c5b41bf1763b3a942282f964f72425ae57) AuthConfig (a9800464506c58f005016df354e160757732171ff4d19747202524955329b998) AuthConfig (0913a452a02695c6681f1037232199b40b64236fed4d29cdbb41561695826340) AuthConfig (2d953f7313807988e63546aa32431d96f77e6989c672f8ab42827c1c21918630) AuthConfig (67c67ce733de50e7b3d797bb248036b01badfe02e958f5f76af05b58a504989c) AuthConfig (2ddbced6f2f51d410743da1d9db1257d0006d99f38986dd7cc7efd26620791ad) AuthConfig (f9b535aed6243496a67741936e53a0d303bea1b872a06fb62f1f9c27150edc40) AuthConfig (319e246831bdaf5e765e63d5511b516594fd2b64527850e2ddabd2a4a85532ea) AuthConfig (8d34bd12ba2030284fcb69127f53b8158bbdd4a1432bfe619165b4c2c96cc061) AuthConfig (d303c788902ad4467c7ae545d65cbef45564d1c0d56396c93057df3909461aa1) AuthConfig (17002b7f0c8e62582ad262d9c2b8b038ee7f2a9327fe34f21fd4f47134b7b8cf) AuthConfig (bf65ad13397c94660867eb0205e06d1c9bebda4b019f632c0d0b8ef989930381) AuthConfig (4d18a96899c43d55ee63d2ae1a7a938c7e873d88b2fd95f06b2e3aae2c7f1c84) AuthConfig (596921ddc7ba8523386263c5cffcbd6ad9b6a385318ed7e24f72be06f36fc398) AuthConfig (105cf4dd91a8abf4a93abd65b255521c3d37b560b5d23f3ec59757b5b362fae6) AuthConfig (c5a685f2a5ebdef5f1dee488990c0798a45ddce399cf99e7d0c505a5f46c69ac) AuthConfig (a6a6232fd5656520e75d100c41add5f1637dde9705a4ec608bbb90b724c258a5) AuthConfig (cc1ffc2ee4e05618c28c108cac668c0c02eb8e658953439e47a733edab44a1a0) AuthConfig (1704c6bc15d6f547acca888061680f2275f690cb79e5af6fee44747bcf278b95) AuthConfig (028a8b5bd5d4e690eebb2f20c5fd47cf0b3c6e2a3619ba9e55322bbca7b94cc3) AuthConfig (a8c8220ad6f1387002e262ace5db78a2c4bb39e5953306702a228215174f6796) AuthConfig (d3dc05ddf6e7ba7b9f9979bea25d3874196895f35b8f4cdf2a9ecdec10126455) AuthConfig (135ffad9bbcd33779a0b9a1c0dc5d5f547356be21489762803640aca855c728a) AuthConfig (ad8714d3b8aed2a9ffd68fbcf0288557e9c1c059c1b59ad3e9a09910b37fd61b) AuthConfig (8d8c8aae1332bb0300ac2d65be9b8708fbfde6d4b5599ec52380cf1f70956045) AuthConfig (fac07f302f653115fda3d376325a268abaf66025ed0e4af73e88bea3f41fe933) AuthConfig (d1d232e06f90d479cd9b802f24889e83af58c51cf54ba273d5e24dcb8a297688) AuthConfig (6565c4085b3917d5390fe1b0194fb3d8319f1d56b45867e3c68ff740b5c16ef3) AuthConfig (6687eb3cec74c6c51428ce3f3d22d50ba269f59015d1bc8f185da49eb095534e) AuthConfig (de97dd5d45c6eb12b4a1112ed1f362d8544427428d2249a399e26ba069d6aadb) AuthConfig (75fd86e2a4870af5fd78f2ad7aa2a9d79aff977b3678d81954b1e207a9315f2c) AuthConfig (c4399a95e89654e5b2bb5fabe45bc947d11ccd1448ebf64c30b195b34f239d58) AuthConfig (b9c6946adb079822c7b75166f990a8c6505dbb8a6487fa707e670ff65560e668) AuthConfig (5d590119308c6f848b12dd3ecdfb1b06cc2fe172af702455c342075944a045ff) AuthConfig (68502d1c1143a647ad56f2759ea89a4c5b3c74a8054739f7e7ea9ba6535e3b74) AuthConfig (23672b974f0dbc05100a34c78d2b029e41f1d5a3752f3fcb52915f56db360e25) AuthConfig (3386c5e86c72abcdab369776ccd7182024ab8681cbe497e81724868f101a3c0c) AuthConfig (75612d85af2bae456dadd78163b2d2f9015c0a0bf4e20bcb8a555e6a5ab3b8c0) AuthConfig (3083301b5359be118f04faaefcd72fd127f2297a3374ef7231b59a59a31b405c) AuthConfig (0829dbf2b5dc0900de272826295adf351de381b46fa6fba00b896213530780f6) AuthConfig (bfd36c5ed73eb81df97258ffa1aa36d8cbf6f3e7a8096c33ca4ff417be59d002) AuthConfig (06165835c4461c0b16a2bc5bae936aff3c2f351edbef8aa378f1110874acedfc) AuthConfig (ab505bdf28287fe00099987c1bde2c2f297c40c792a7f6cb5fbfa5adc44754d3) AuthConfig (f95805f0e9b755664735d898c1d644d571943be553204d3f86136bc35719ff13) AuthConfig (486615827938fa99814e929fe7dc62bc16f7563179fef855074eb73dc7ce4724) AuthConfig (50c2058cf11e07a931b06ac4446a56eee86855f00badf26618abe486a7bdcfd7) AuthConfig (94ddc5e77f082c32bebc7a36bcebf621518ebf303e83b9c540da1352d04e620b) AuthConfig (669658556fba0f45c2a82def2f57614416614ddf154ed8dc5f4f9188f10130f2) AuthConfig (a402d1681ab35bace735c19b1d084a86eb3d741d4e1c14371cfdaa55d22c3ebe) AuthConfig (ce7d381faee9ff949d68d825ffa1dcb9247dad19667ec59a8e91847320dacbe0) AuthConfig (aa7038625b8f43b0c002f79637065e5ab90468f50f998759d08f2c4640c07c04) AuthConfig (f6df6bc14432fdc346191db86ee06e75c8090ed87a32c984493c837e97a10be6) AuthConfig (74b47f1aa1203aad58596cbaab10ae46e8fddd08ac49eb57a26bdba532d88e72) AuthConfig (a4493fc88328eba553ef36791fbf4706bffae04a1ac501d099b12309ebc85c6b) AuthConfig (8d5571efc1d304f5bcfc2ffb4ac6819154ba5556c29ce33416d460b887427984) AuthConfig (5e4821ff903ad6ca2e1c42162432e34d8b0a2a78052961e753d36b742193432f) AuthConfig (54eea38241838159e47386cb814ad20fc30631ee8c42d57f0ff8da80fbbf62a0) AuthConfig (b64d1d3a1a41de71c5b05aa58c2da5025409dfbb29fbaa63fe6a7b8fb71bf42e) AuthConfig (726a74c13fee356e859d9317271facbb60b066acbc5571de22d056f13c60878f) AuthConfig (2416d7b42d11a09d7152746be0e114f8fdc87d0fbd0efa9dec5b52b011082194) AuthConfig (e91aceffc2d4f6d6b120625bd3ca23781b74a1cae6cf1da26bda310bbed0ef07) AuthConfig (dc061bec60bd28b1c132bf30965b4810124429576337bcc4a6030617600d1714) AuthConfig (99606c4eac496429be47144db9f3f48867815df12eb99d7be8073b767d7e44de) AuthConfig (54cd0149b818480305f3d0024b3c95629cafb641b48195d003b2b7006d5c6bcd) AuthConfig (1e077a716d89a43572a3c16e8d0b0436816baa48a2ea227603701f10cad6d6f4) AuthConfig (0e042276580ed8976d8ad3afd87a71640025c28b9b48f5c88eb159ec9998209f) AuthConfig (5c6918972d1a7f8ad5201880b7d6d9c1191ca1f25376682f691def7868e3fd02) AuthConfig (c82a1a3dc2ce54c0b664f79a2cd895a3c40f774b3694999badae8cedd328292e) AuthConfig (80056c658070f49140955cb6eabdbd252857a614af6b0f2f2de7a1ab1cd47949) AuthConfig (dd89509e97b96670685602ae2d08749226db8693944d2c6f868af9d3e0a5feb5) AuthConfig (3167d3bf853032a93e8569019dcd3e5aa33af9cb580cf9ca138990567fc9d857) AuthConfig (7def4ca304ff3869be64c2c02523fd9d2ff43465099dcd1124399f293eb4480f) AuthConfig (f9fa9cfc2b4e701183573419a92c79384cd30210289602f0115c75c5029d55aa) AuthConfig (a908063c86927e6bf07b7d793aad9f277ac3c2a8f5f5d468b116cb9e07929b14) AuthConfig (010adf02b2e7deb40c572f4729b6dd4619847c8e6344137326b72df28772fc1a) AuthConfig (4e5c1437f299959e62fd8c200a351f13f27078882a4684be54f26facece20f96) AuthConfig (8274337fc3f0de48e5213819aeb4a46559c4f8b3428c1bae43c0d75207ef5264) AuthConfig (2d4a2275e6cbab5c26d5c3cf2d75aa2776066390e5b66f5498b16b26bc596af7) AuthConfig (06b05a8e4dd69bdc4f333ad9509ee8cad3c5c0f5dd48edbe94c1158b60f79ed3) AuthConfig (efc66033da0da4dfd829f59f79b4f7de512c24b6b150aa338f2a963b5d74b7ce) AuthConfig (92917da87eaba4f2684cd45947a3698434b99ff4ca6d3ddde51d6573aecb1600) AuthConfig (810f299b76daf089b33c92cfbe0445ab5cc89a48d38e36df0cc89ed6cd1fe8df) AuthConfig (3f23315aacd6e16f4ab41f96aeedc9549d1abdfc9476577a47d45d12875dec5a) AuthConfig (d69b9c128f599c9ea1af84bb3e783ed71688baf6b3ad2b86f2b7183f9a4208b9)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.gateway_host = 'maas.apps.64a0b6d086e4bf8d4499.prod.konflux-ocp-ci.dev' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, tenant_b_external_oidc_from_env, ) require_aitenant_crd() worker = _xdist_worker_suffix() case_a = new_named_tenant_case(f"e2e-shared-a-{worker}") case_b = new_named_tenant_case(f"e2e-shared-b-{worker}") try: # Bootstrap both tenants (creates gateway + AITenant CR) bootstrap_aitenant_tenant(case_a) > bootstrap_aitenant_tenant( case_b, external_oidc=tenant_b_external_oidc_from_env() ) test/maas-e2e/test/e2e/tests/conftest.py:527: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-shared-b-w2-c4bbec', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f2deb57e700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-shared-b-w2-c4bbec in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-shared-b-w2-c4bbec","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-shared-b-w2-c4bbec"}}}\n'}, 'creationTimestamp': '2026-11-01T12:51:53Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-shared-b-w2-c4bbec', 'namespace': 'ai-tenants', 'resourceVersion': '56484', 'uid': '4f06f15b-72a5-4dbd-ad04-c94614cbd102'}, 'spec': {'gateway': {'name': 'e2e-shared-b-w2-c4bbec'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:51:56Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-shared-b-w2-c4bbec', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-shared-b-w2-c4bbec'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-a-6b9030', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f4733986700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-a-6b9030 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-a-6b9030","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-a-6b9030"}}}\n'}, 'creationTimestamp': '2026-11-01T12:53:48Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-a-6b9030', 'namespace': 'ai-tenants', 'resourceVersion': '59344', 'uid': '39a74449-04a7-4bcd-8105-ae081d7d6790'}, 'spec': {'gateway': {'name': 'e2e-api-a-6b9030'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:53:50Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-a-6b9030', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-a-6b9030'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-a-f893c2', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7faaff3b7560>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-a-f893c2 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-a-f893c2","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-a-f893c2"}}}\n'}, 'creationTimestamp': '2026-11-01T12:52:49Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-a-f893c2', 'namespace': 'ai-tenants', 'resourceVersion': '57673', 'uid': '78c9470c-1c9e-4c6d-bd87-7c05e2ae4c16'}, 'spec': {'gateway': {'name': 'e2e-api-a-f893c2'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:52:53Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-a-f893c2', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-a-f893c2'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-a-f893c2', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7faaff3b7560>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-a-f893c2 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-a-f893c2","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-a-f893c2"}}}\n'}, 'creationTimestamp': '2026-11-01T12:52:49Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-a-f893c2', 'namespace': 'ai-tenants', 'resourceVersion': '57673', 'uid': '78c9470c-1c9e-4c6d-bd87-7c05e2ae4c16'}, 'spec': {'gateway': {'name': 'e2e-api-a-f893c2'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:52:53Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-a-f893c2', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-a-f893c2'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-0a0610', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f1dc0dfb2e0>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-0a0610 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-0a0610","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-0a0610"}}}\n'}, 'creationTimestamp': '2026-11-01T12:54:25Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-0a0610', 'namespace': 'ai-tenants', 'resourceVersion': '60318', 'uid': 'c8082fe0-3ee8-4293-b981-d28659194908'}, 'spec': {'gateway': {'name': 'e2e-inf-a-0a0610'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:54:27Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-0a0610', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-0a0610'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-0a0610', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f1dc0dfb2e0>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-0a0610 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-0a0610","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-0a0610"}}}\n'}, 'creationTimestamp': '2026-11-01T12:54:25Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-0a0610', 'namespace': 'ai-tenants', 'resourceVersion': '60318', 'uid': 'c8082fe0-3ee8-4293-b981-d28659194908'}, 'spec': {'gateway': {'name': 'e2e-inf-a-0a0610'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:54:27Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-0a0610', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-0a0610'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorgateway_host = 'maas.apps.64a0b6d086e4bf8d4499.prod.konflux-ocp-ci.dev' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, tenant_b_external_oidc_from_env, ) require_aitenant_crd() worker = _xdist_worker_suffix() case_a = new_named_tenant_case(f"e2e-shared-a-{worker}") case_b = new_named_tenant_case(f"e2e-shared-b-{worker}") try: # Bootstrap both tenants (creates gateway + AITenant CR) bootstrap_aitenant_tenant(case_a) > bootstrap_aitenant_tenant( case_b, external_oidc=tenant_b_external_oidc_from_env() ) test/maas-e2e/test/e2e/tests/conftest.py:527: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-shared-b-w4-78c69d', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f3f666a6700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-shared-b-w4-78c69d in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-shared-b-w4-78c69d","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-shared-b-w4-78c69d"}}}\n'}, 'creationTimestamp': '2026-11-01T12:54:41Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-shared-b-w4-78c69d', 'namespace': 'ai-tenants', 'resourceVersion': '60903', 'uid': 'f80a3260-78f3-44e8-810a-90f007d2be14'}, 'spec': {'gateway': {'name': 'e2e-shared-b-w4-78c69d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:54:43Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-shared-b-w4-78c69d', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-shared-b-w4-78c69d'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-b-2810c2', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f868fa8a700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-b-2810c2 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-b-2810c2","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-b-2810c2"}}}\n'}, 'creationTimestamp': '2026-11-01T12:54:56Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-b-2810c2', 'namespace': 'ai-tenants', 'resourceVersion': '61389', 'uid': '9d1bea3e-2f3b-4fbf-897d-2fde1e596d28'}, 'spec': {'gateway': {'name': 'e2e-api-b-2810c2'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:54:58Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-b-2810c2', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-b-2810c2'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-b-2810c2', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f868fa8a700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-b-2810c2 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-b-2810c2","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-b-2810c2"}}}\n'}, 'creationTimestamp': '2026-11-01T12:54:56Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-b-2810c2', 'namespace': 'ai-tenants', 'resourceVersion': '61389', 'uid': '9d1bea3e-2f3b-4fbf-897d-2fde1e596d28'}, 'spec': {'gateway': {'name': 'e2e-api-b-2810c2'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-01T12:54:58Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-b-2810c2', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-b-2810c2'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError