/workspace/source/test/e2e/tests/test_api_keys.py:930: CronJob maas-api-key-cleanup not found in opendatahub: Error from server (NotFound): cronjobs.batch "maas-api-key-cleanup" not found/workspace/source/test/e2e/tests/test_api_keys.py:977: NetworkPolicy maas-api-cleanup-restrict not found in opendatahub: Error from server (NotFound): networkpolicies.networking.k8s.io "maas-api-cleanup-restrict" not found/workspace/source/test/e2e/tests/test_api_keys.py:1096: Cannot find maas-api pod in opendatahub: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not assertedself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7fc83446f4f0> def test_aitenant_delete_cleans_up_bootstrap_resources(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) _apply_aitenant(case) > _assert_aitenant_bootstrap_resources(case) test/e2e/tests/test_aitenant_lifecycle.py:389: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:197: in _assert_aitenant_bootstrap_resources aitenant = _wait_for_json( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ait-cc97ffc3', namespace = 'ai-tenants' def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: aitenant/e2e-ait-cc97ffc3 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ait-cc97ffc3","namespace":"ai-tenants"},"spec":{}}\n'}, 'creationTimestamp': '2026-07-08T17:47:39Z', 'generation': 1, 'name': 'e2e-ait-cc97ffc3', 'namespace': 'ai-tenants', 'resourceVersion': '41807', 'uid': 'e54bf1c5-753f-42a6-9e5c-679833040c81'}, 'spec': {}} test/e2e/tests/test_aitenant_lifecycle.py:109: AssertionErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7fc83446f070> def test_aitenant_derives_non_default_tenant_namespace(self): """RHOAIENG-66836: non-default AITenant must not use models-as-a-service tenant namespace.""" suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-derive-{suffix}" reserved_ns = os.environ.get("MAAS_SUBSCRIPTION_NAMESPACE", "models-as-a-service") expected_ns = f"ai-tenant-{aitenant_name}" gateway_name = aitenant_name try: _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "AITenant", "metadata": {"name": aitenant_name, "namespace": AITENANT_NAMESPACE}, "spec": {}, } ) > aitenant = _wait_for_json( AITENANT_KIND, aitenant_name, AITENANT_NAMESPACE, predicate=_aitenant_ready, timeout=120, ) test/e2e/tests/test_aitenant_lifecycle.py:423: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-derive-9ad3129d', namespace = 'ai-tenants' def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: aitenant/e2e-derive-9ad3129d in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-derive-9ad3129d","namespace":"ai-tenants"},"spec":{}}\n'}, 'creationTimestamp': '2026-07-08T17:50:44Z', 'generation': 1, 'name': 'e2e-derive-9ad3129d', 'namespace': 'ai-tenants', 'resourceVersion': '44040', 'uid': '4b263c61-04a9-48f2-9e38-620739ca1b64'}, 'spec': {}} test/e2e/tests/test_aitenant_lifecycle.py:109: AssertionError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7fc834586700> def test_tenant_admin_rbac_is_namespace_scoped(self): """1.6: Tenant-admin Role from AITenant bootstrap is scoped to the tenant namespace.""" case = new_discovery_case() sa_name = f"e2e-mt-rbac-{case['suffix']}" role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" other_ns = f"e2e-mt-other-{case['suffix']}" try: apply_gateway_fixture(case["gateway_name"], fixture_label=case["tenant_label_name"]) > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_namespace_discovery.py:260: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:764: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-mt-9f98ee07', namespace = 'ai-tenants' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-mt-9f98ee07 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-mt-9f98ee07","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-mt-9f98ee07"}}}\n'}, 'creationTimestamp': '2026-07-08T17:54:27Z', 'generation': 1, 'name': 'e2e-mt-9f98ee07', 'namespace': 'ai-tenants', 'resourceVersion': '46996', 'uid': '22425a86-dc45-4cf8-82fe-dcdc8a16175a'}, 'spec': {'gateway': {'name': 'e2e-mt-9f98ee07'}}} test/e2e/tests/multitenancy_helpers.py:227: AssertionError/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7fc83446f820> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: > bootstrap_aitenant_tenant(case) test/e2e/tests/test_multi_tenant_integration.py:74: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:764: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-mt-16a28f03', namespace = 'ai-tenants' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-mt-16a28f03 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-mt-16a28f03","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-mt-16a28f03"}}}\n'}, 'creationTimestamp': '2026-07-08T17:57:54Z', 'generation': 1, 'name': 'e2e-mt-16a28f03', 'namespace': 'ai-tenants', 'resourceVersion': '49465', 'uid': '21dd5611-91fc-4904-a939-022aaa8c7037'}, 'spec': {'gateway': {'name': 'e2e-mt-16a28f03'}}} test/e2e/tests/multitenancy_helpers.py:227: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_model_inference.py:77: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:764: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-5227d3', namespace = 'ai-tenants' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-5227d3 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-5227d3","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-5227d3"}}}\n'}, 'creationTimestamp': '2026-07-08T18:01:57Z', 'generation': 1, 'name': 'e2e-inf-a-5227d3', 'namespace': 'ai-tenants', 'resourceVersion': '52458', 'uid': '8ec6f175-2a3a-4f91-a0f7-753b15658c64'}, 'spec': {'gateway': {'name': 'e2e-inf-a-5227d3'}}} test/e2e/tests/multitenancy_helpers.py:227: AssertionError