/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.maas_api_base_url = 'https://maas.apps.67cc2d84bceb8ee80679.prod.konflux-ocp-ci.dev/maas-api' def test_tokens_endpoint_replaced_by_api_keys(maas_api_base_url: str): """ Verify /v1/tokens endpoint has been replaced by the API key system. The endpoint may return 401 (auth required), 404 (removed), or 405 (method not allowed). Any of these indicates the old token minting is no longer available. """ url = f"{maas_api_base_url}/v1/tokens" r = requests.post(url, json={"expiration": "1m"}, timeout=20, verify=TLS_VERIFY) msg = f"[token] POST {url} (no auth) -> {r.status_code}" log.info(msg) print(msg) # 401/404/405 all indicate the old token minting endpoint is gone/protected > assert r.status_code in (401, 404, 405), f"Expected 401/404/405 (endpoint replaced), got {r.status_code}: {r.text[:400]}" E AssertionError: Expected 401/404/405 (endpoint replaced), got 500: Internal Server Error. E E assert 500 in (401, 404, 405) E + where 500 = <Response [500]>.status_code test/maas-e2e/test/e2e/tests/test_smoke.py:50: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks object at 0x7fbe70605650> def test_models_as_a_service_namespace_reconciles(self): ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-default-ns-{suffix}" try: apply_maas_auth_policy(policy_name, ns) wait_for_finalizer("maasauthpolicy", policy_name, ns, FINALIZER_AUTHPOLICY) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:394: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-ns-bdf94388' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fbe709754e0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-ns-bdf94388 in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-ns-bdf94388","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-03T19:47:30Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-ns-bdf94388', 'namespace': 'models-as-a-service', 'resourceVersion': '59755', 'uid': '70f75a91-d9a6-4265-8f7a-944499495068'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-03T19:47:30Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 38', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7ff42dd9fb10> def test_labeled_tenant_namespace_is_discovered(self): """1.1: Controller discovers labeled tenant namespaces and reconciles CRs.""" case = new_discovery_case(use_default_gateway=True) try: apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", case["subscription_name"], case["tenant_ns"], FINALIZER_SUBSCRIPTION) > auth = wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-b9eb2205' namespace = 'ai-tenant-e2e-mt-b9eb2205' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7ff42d8be980> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-b9eb2205 in ai-tenant-e2e-mt-b9eb2205 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-b9eb2205","namespace":"ai-tenant-e2e-mt-b9eb2205"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-03T19:48:00Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-b9eb2205', 'namespace': 'ai-tenant-e2e-mt-b9eb2205', 'resourceVersion': '60845', 'uid': '0f041569-f009-440c-8d79-fe5c068ccc87'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-03T19:48:00Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 41', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gatewayself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f16c195ae50> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase="Active", ) wait_for_aitenant_cleanup_resources(case) # Bind the key to the tenant's subscription, then verify it remains # invalid after the tenant and its subscription are recreated. route = wait_for_route_admitted(f"{case['gateway_name']}-route") tenant_api_url = f"https://{route['spec']['host']}/maas-api" wait_for_deployment_available(f"maas-api-{case['tenant_label_name']}", timeout=180) key_response = create_api_key_at( tenant_api_url, _get_cluster_token(), f"e2e-aitenant-delete-{case['suffix']}", subscription=case["subscription_name"], ) > assert key_response.status_code in (200, 201), ( f"failed to create tenant-scoped API key: {response_summary(key_response)}" ) E AssertionError: failed to create tenant-scoped API key: status=503 x-ext-auth-reason=None body= E assert 503 in (200, 201) E + where 503 = <Response [503]>.status_code test/maas-e2e/test/e2e/tests/test_multi_tenant_integration.py:136: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-2747a4', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f062cd86700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-2747a4 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-2747a4","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-2747a4"}}}\n'}, 'creationTimestamp': '2026-11-03T19:57:27Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-2747a4', 'namespace': 'ai-tenants', 'resourceVersion': '84812', 'uid': '370d60ed-3831-4195-9589-35e55701d8d9'}, 'spec': {'gateway': {'name': 'e2e-inf-a-2747a4'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-03T19:57:28Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-2747a4', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-2747a4'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1119: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-a-2747a4', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f062cd86700>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-a-2747a4 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-a-2747a4","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-a-2747a4"}}}\n'}, 'creationTimestamp': '2026-11-03T19:57:27Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-a-2747a4', 'namespace': 'ai-tenants', 'resourceVersion': '84812', 'uid': '370d60ed-3831-4195-9589-35e55701d8d9'}, 'spec': {'gateway': {'name': 'e2e-inf-a-2747a4'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-03T19:57:28Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-a-2747a4', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-a-2747a4'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B