/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7fb888194f10> def test_aitenant_derives_non_default_tenant_namespace(self): """RHOAIENG-66836: non-default AITenant must not use models-as-a-service tenant namespace.""" suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-derive-{suffix}" reserved_ns = _ns() expected_ns = f"ai-tenant-{aitenant_name}" gateway_name = aitenant_name try: _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "AITenant", "metadata": {"name": aitenant_name, "namespace": AITENANT_NAMESPACE}, "spec": {}, } ) > aitenant = _wait_for_json( AITENANT_KIND, aitenant_name, AITENANT_NAMESPACE, predicate=_aitenant_ready, timeout=120, ) test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:689: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-derive-fb2343fe', namespace = 'ai-tenants' predicate = <function _aitenant_ready at 0x7fb88878e0c0>, timeout = 120 interval = 5 def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: aitenant/e2e-derive-fb2343fe in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-derive-fb2343fe","namespace":"ai-tenants"},"spec":{}}\n'}, 'creationTimestamp': '2026-11-02T21:56:20Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-derive-fb2343fe', 'namespace': 'ai-tenants', 'resourceVersion': '44350', 'uid': '5784ad85-1930-4d01-a73a-b65c43574ec3'}, 'spec': {}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T21:56:24Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-derive-fb2343fe', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-derive-fb2343fe'}} test/maas-e2e/test/e2e/tests/test_aitenant_lifecycle.py:170: AssertionErrorself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7fa2431c7c90> def test_dynamic_discovery_after_label_added(self): """1.2 variant: Pre-existing CRs reconcile after namespace gains discovery labels.""" case = new_discovery_case(use_default_gateway=True) try: ensure_namespace(case["tenant_ns"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) time.sleep(10) before = get_json_or_none("maasauthpolicy", case["policy_name"], case["tenant_ns"]) assert before is not None assert FINALIZER_AUTHPOLICY not in ((before.get("metadata") or {}).get("finalizers") or []) apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) > wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-47a87a57' namespace = 'ai-tenant-e2e-mt-47a87a57' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fa2431a2f20> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-47a87a57 in ai-tenant-e2e-mt-47a87a57 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-47a87a57","namespace":"ai-tenant-e2e-mt-47a87a57"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T21:56:23Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-47a87a57', 'namespace': 'ai-tenant-e2e-mt-47a87a57', 'resourceVersion': '51187', 'uid': 'dc12d5da-1459-4e14-9254-a39971bce459'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T21:56:34Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 4, current generation is 37', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks object at 0x7f2573286ad0> def test_models_as_a_service_namespace_reconciles(self): ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-default-ns-{suffix}" try: apply_maas_auth_policy(policy_name, ns) wait_for_finalizer("maasauthpolicy", policy_name, ns, FINALIZER_AUTHPOLICY) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:394: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-ns-cfb265d1' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f2572fe18a0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-ns-cfb265d1 in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-ns-cfb265d1","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T21:56:48Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-ns-cfb265d1', 'namespace': 'models-as-a-service', 'resourceVersion': '51739', 'uid': '49fd85ce-43bd-400f-8b35-f547ffc598cc'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T21:56:48Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 4, current generation is 40', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gatewayself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f0cc4ffc750> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase="Active", ) wait_for_aitenant_cleanup_resources(case) # Bind the key to the tenant's subscription, then verify it remains # invalid after the tenant and its subscription are recreated. route = wait_for_route_admitted(f"{case['gateway_name']}-route") tenant_api_url = f"https://{route['spec']['host']}/maas-api" wait_for_deployment_available(f"maas-api-{case['tenant_label_name']}", timeout=180) key_response = create_api_key_at( tenant_api_url, _get_cluster_token(), f"e2e-aitenant-delete-{case['suffix']}", subscription=case["subscription_name"], ) assert key_response.status_code in (200, 201), ( f"failed to create tenant-scoped API key: {response_summary(key_response)}" ) old_api_key = key_response.json().get("key") assert old_api_key, "tenant-scoped API key response did not include a key" validation = validate_api_key_at( tenant_internal_url(case["tenant_label_name"]), old_api_key, ) assert validation.status_code == 200 and validation.json().get("valid") is True, ( f"new tenant API key did not validate: status={validation.status_code} " f"body={validation.text}" ) user_objects = apply_unrelated_tenant_objects(case) delete_best_effort(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout="180s") wait_for_not_found(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) wait_for_not_found("maasauthpolicy", case["policy_name"], case["tenant_ns"], timeout=180) wait_for_not_found("maassubscription", case["subscription_name"], case["tenant_ns"], timeout=180) wait_for_not_found("role", role_name, case["tenant_ns"], timeout=180) namespace = wait_for_json( "namespace", case["tenant_ns"], predicate=lambda obj: LABEL_MANAGED_BY_AITENANT not in ((obj.get("metadata") or {}).get("labels") or {}), timeout=180, ) namespace_metadata = namespace.get("metadata") or {} namespace_labels = namespace_metadata.get("labels") or {} namespace_annotations = namespace_metadata.get("annotations") or {} for key in ( "app.kubernetes.io/managed-by", "app.kubernetes.io/part-of", LABEL_AI_GATEWAY_TENANT, LABEL_MANAGED_BY_AITENANT, LABEL_TENANT_NAME, LABEL_TENANT_NAMESPACE, "opendatahub.io/generated-namespace", ): assert key not in namespace_labels for key in ( ANNOTATION_AITENANT_NAME, ANNOTATION_AITENANT_NAMESPACE, "maas.opendatahub.io/created-by-aitenant", ): assert key not in namespace_annotations assert get_json_or_none("secret", user_objects["secret"], case["tenant_ns"]) is not None assert get_json_or_none("rolebinding", user_objects["rolebinding"], case["tenant_ns"]) is not None wait_for_aitenant_cleanup_resources_deleted(case) # Recreate the same tenant name and verify the old key cannot be # used against the newly provisioned maas-api instance. > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_integration.py:189: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-mt-e2be7cc3', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f0cc5893560>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-mt-e2be7cc3 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-mt-e2be7cc3","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-mt-e2be7cc3"}}}\n'}, 'creationTimestamp': '2026-11-02T22:06:57Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-mt-e2be7cc3', 'namespace': 'ai-tenants', 'resourceVersion': '65188', 'uid': '3067f701-7d29-4629-bc76-3c470ff8d61d'}, 'spec': {'gateway': {'name': 'e2e-mt-e2be7cc3'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:07:00Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-mt-e2be7cc3', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-mt-e2be7cc3'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.gateway_host = 'maas.apps.0f8d2781715f7880749e.prod.konflux-ocp-ci.dev' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, tenant_b_external_oidc_from_env, ) require_aitenant_crd() worker = _xdist_worker_suffix() case_a = new_named_tenant_case(f"e2e-shared-a-{worker}") case_b = new_named_tenant_case(f"e2e-shared-b-{worker}") try: # Bootstrap both tenants (creates gateway + AITenant CR) bootstrap_aitenant_tenant(case_a) > bootstrap_aitenant_tenant( case_b, external_oidc=tenant_b_external_oidc_from_env() ) test/maas-e2e/test/e2e/tests/conftest.py:527: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-shared-b-w2-3eb190', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7ffbd0bef560>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-shared-b-w2-3eb190 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-shared-b-w2-3eb190","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-shared-b-w2-3eb190"}}}\n'}, 'creationTimestamp': '2026-11-02T22:04:05Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-shared-b-w2-3eb190', 'namespace': 'ai-tenants', 'resourceVersion': '60791', 'uid': 'fc240c96-f2d9-4d3f-8de6-85c0727e1a36'}, 'spec': {'gateway': {'name': 'e2e-shared-b-w2-3eb190'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:04:07Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-shared-b-w2-3eb190', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-shared-b-w2-3eb190'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-a-224f53', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7fa24378e700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-a-224f53 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-a-224f53","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-a-224f53"}}}\n'}, 'creationTimestamp': '2026-11-02T22:04:39Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-a-224f53', 'namespace': 'ai-tenants', 'resourceVersion': '61728', 'uid': '1a646ae6-b241-4f86-ad6d-330754e58317'}, 'spec': {'gateway': {'name': 'e2e-api-a-224f53'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:04:41Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-a-224f53', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-a-224f53'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-a-588b5c', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f4715396700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-a-588b5c in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-a-588b5c","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-a-588b5c"}}}\n'}, 'creationTimestamp': '2026-11-02T22:06:58Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-a-588b5c', 'namespace': 'ai-tenants', 'resourceVersion': '65409', 'uid': '37eda96b-edce-4b4e-a441-9f2029b362dc'}, 'spec': {'gateway': {'name': 'e2e-api-a-588b5c'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:07:01Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-a-588b5c', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-a-588b5c'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-2050f3', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f6652ed6700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-2050f3 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-2050f3","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-2050f3"}}}\n'}, 'creationTimestamp': '2026-11-02T22:07:14Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-2050f3', 'namespace': 'ai-tenants', 'resourceVersion': '65889', 'uid': '7a68cb9f-0232-4cd5-a3ca-de4bc345f9d4'}, 'spec': {'gateway': {'name': 'e2e-inf-a-2050f3'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:07:16Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-2050f3', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-2050f3'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-2050f3', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f6652ed6700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-2050f3 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-2050f3","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-2050f3"}}}\n'}, 'creationTimestamp': '2026-11-02T22:07:14Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-2050f3', 'namespace': 'ai-tenants', 'resourceVersion': '65889', 'uid': '7a68cb9f-0232-4cd5-a3ca-de4bc345f9d4'}, 'spec': {'gateway': {'name': 'e2e-inf-a-2050f3'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:07:16Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-2050f3', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-2050f3'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-b-587629', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7fb8888972e0>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-b-587629 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-b-587629","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-b-587629"}}}\n'}, 'creationTimestamp': '2026-11-02T22:04:52Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-b-587629', 'namespace': 'ai-tenants', 'resourceVersion': '62215', 'uid': '9ff12253-7372-47eb-a813-edc646397537'}, 'spec': {'gateway': {'name': 'e2e-api-b-587629'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:04:53Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-b-587629', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-b-587629'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-b-587629', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7fb8888972e0>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-b-587629 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-b-587629","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-b-587629"}}}\n'}, 'creationTimestamp': '2026-11-02T22:04:52Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-b-587629', 'namespace': 'ai-tenants', 'resourceVersion': '62215', 'uid': '9ff12253-7372-47eb-a813-edc646397537'}, 'spec': {'gateway': {'name': 'e2e-api-b-587629'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:04:53Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-b-587629', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-b-587629'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-17435d', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f25736c2700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-17435d in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-17435d","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-17435d"}}}\n'}, 'creationTimestamp': '2026-11-02T22:08:22Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-17435d', 'namespace': 'ai-tenants', 'resourceVersion': '67670', 'uid': '3fcab280-592d-4f10-8731-d703646e4697'}, 'spec': {'gateway': {'name': 'e2e-inf-a-17435d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:08:25Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-17435d', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-17435d'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-17435d', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f25736c2700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-17435d in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-17435d","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-17435d"}}}\n'}, 'creationTimestamp': '2026-11-02T22:08:22Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-17435d', 'namespace': 'ai-tenants', 'resourceVersion': '67670', 'uid': '3fcab280-592d-4f10-8731-d703646e4697'}, 'spec': {'gateway': {'name': 'e2e-inf-a-17435d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:08:25Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-17435d', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-17435d'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_cases(): require_aitenant_crd() case_a = new_named_tenant_case("e2e-api-a") case_b = new_named_tenant_case("e2e-api-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:58: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-api-a-bdc145', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7ffbd0bef560>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-api-a-bdc145 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-api-a-bdc145","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-api-a-bdc145"}}}\n'}, 'creationTimestamp': '2026-11-02T22:09:00Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-api-a-bdc145', 'namespace': 'ai-tenants', 'resourceVersion': '68809', 'uid': 'e46f0251-1e32-457a-b04e-256765aab796'}, 'spec': {'gateway': {'name': 'e2e-api-a-bdc145'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:09:02Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-api-a-bdc145', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-api-a-bdc145'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-027bdf', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7fa24378e700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-027bdf in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-027bdf","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-027bdf"}}}\n'}, 'creationTimestamp': '2026-11-02T22:09:34Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-027bdf', 'namespace': 'ai-tenants', 'resourceVersion': '69953', 'uid': '00221347-11df-4f56-b2db-32204ad3cccc'}, 'spec': {'gateway': {'name': 'e2e-inf-a-027bdf'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T22:09:36Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-027bdf', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-027bdf'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError