/workspace/source/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gatewayself = <test_models_endpoint.TestModelsEndpoint object at 0x7f6b509ebd10> def test_unauthenticated_request_401(self): """ Test: Request to /v1/models without Authorization header gets 401. Expected: HTTP 401 (authentication_error). """ # Test: GET /v1/models WITHOUT Authorization header # Expected: 401 Unauthorized log.info("Testing: GET /v1/models without Authorization header") url = f"{_maas_api_url()}/v1/models" r = requests.get( url, timeout=TIMEOUT, verify=TLS_VERIFY, ) > assert r.status_code == 401, f"Expected 401 for unauthenticated request, got {r.status_code}: {r.text}" E AssertionError: Expected 401 for unauthenticated request, got 500: Internal Server Error. E E assert 500 == 401 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_models_endpoint.py:2216: AssertionErrorcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w0-e57216' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_models_endpoint.py:381: Failedcls = <class 'test_models_endpoint.TestModelsEndpoint'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests.""" log.info("=" * 60) log.info("Validating /v1/models E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found in namespace '{MODEL_NAMESPACE}'. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w0-e57216' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_models_endpoint.py:381: Failed/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestMissingModelRef object at 0x7fdde7b3d6d0> def test_authpolicy_with_nonexistent_model_ref(self): """MaaSAuthPolicy does not generate per-model AuthPolicies in gateway-only mode. Creates an auth policy referencing one valid model and one ghost model, waits for Degraded phase, then asserts that no legacy per-model AuthPolicies exist for either model. """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-ghost-policy-{suffix}" ghost_model = f"nonexistent-model-{suffix}" try: _create_test_auth_policy( policy_name, [MODEL_REF, ghost_model], groups=["system:authenticated"], ) > _wait_for_maas_auth_policy_phase(policy_name, "Degraded", timeout=60, require_auth_policies=False) test/e2e/tests/test_negative_security.py:578: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-neg-ghost-policy-82f5834f', expected_phase = 'Degraded' namespace = 'models-as-a-service', timeout = 60, require_auth_policies = False require_enforced = True def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=AUTHPOLICY_PHASE_TIMEOUT, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") last_snapshot = "not found" while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr is None: last_snapshot = "not found" else: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) ready_count = sum(1 for ap in auth_policies if ap.get("ready") is True) last_snapshot = f"phase={phase}, authPolicies={len(auth_policies)} (ready={ready_count})" if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: {last_snapshot}") time.sleep(2) # Report what the loop last saw: a re-read after the deadline can show a # state the loop never evaluated. > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(last seen: {last_snapshot})" ) E TimeoutError: MaaSAuthPolicy e2e-neg-ghost-policy-82f5834f did not reach phase 'Degraded' within 60s (last seen: phase=Pending, authPolicies=0 (ready=0)) test/e2e/tests/test_helper.py:1889: TimeoutError/workspace/source/test/e2e/tests/test_networkpolicy.py:119: AIGC praxis NetworkPolicy does not include OTLP 4317 egress (MaaS Go IPP bundle only)/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)/workspace/source/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:199: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:227: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:251: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:268: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:307: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:341: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:396: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:434: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_shared.py:165: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-shr-user-6bb225/e2e-shr-shared-384e87-e2e-extmod-a-e1769a status.resolvedModelAlias='e2e-shr-shared-384e87' != 'e2e-shr-shared-384e87-client'/workspace/source/test/e2e/tests/test_tenant_external_model_shared.py:165: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-shr-user-eba670/e2e-shr-quota-735a79-e2e-extmod-a-6f162a status.resolvedModelAlias='e2e-shr-quota-735a79' != 'e2e-shr-quota-735a79-client'/workspace/source/test/e2e/tests/test_tenant_external_model_conflicts.py:362: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-cfl-2-2aa680/e2e-cfl-same-32cb67 status.resolvedModelAlias='e2e-cfl-same-32cb67' != 'e2e-cfl-same-32cb67-two'/workspace/source/test/e2e/tests/test_tenant_external_model_shared.py:165: client-facing model names are not resolvable through the gateway yet: MaaSModelRef e2e-extmod-shr-user-6bb225/e2e-shr-auth-833fef-e2e-extmod-a-e1769a status.resolvedModelAlias='e2e-shr-auth-833fef' != 'e2e-shr-auth-833fef-client'@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:1242: in make_tenant_model_accessible wait_for_status_phase( test/e2e/tests/multitenancy_helpers.py:324: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-90f683-auth' namespace = 'ai-tenant-e2e-inf-a-90f683' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f54b6514ae0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-90f683-auth in ai-tenant-e2e-inf-a-90f683 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-90f683-auth","namespace":"ai-tenant-e2e-inf-a-90f683"},"spec":{"modelRefs":[{"name":"test-model-90f683","namespace":"ai-tenant-e2e-inf-a-90f683"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-29T23:36:17Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-90f683-auth', 'namespace': 'ai-tenant-e2e-inf-a-90f683', 'resourceVersion': '152507', 'uid': '3ba68066-4273-4bbb-8f50-7fa67c7737cb'}, 'spec': {'modelRefs': [{'name': 'test-model-90f683', 'namespace': 'ai-tenant-e2e-inf-a-90f683'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-29T23:36:17Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-90f683-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (61ceb1cbbddc83dc49cabb63dfbe82ce57108c09488c38b284d3a9c12882a181) AuthConfig (e81da58c401266637bb649e74b4d74bbf7a2b4aa75d0277aa0ab96625f9712c5) AuthConfig (a58cb8bcc1268a079aacc0f127a19087d2995c585dca7e812c6eb668b76428f4) AuthConfig (af0b8a3821de4a24044cba30bb35af2b90fea05d98c5fc98b6710881a45c1476) AuthConfig (172db0cf2837c4747bc23ebf9d56e91fa3df7b6a251672bac83b084b19574005) AuthConfig (8a1b50732a941e5c62158e1e23136121dfafdcafd3444ccc3ca4f5e0760858af) AuthConfig (c4b690b8b4b7bf7270db0a65e96ca915e68fc2e2d80a7fb07375cc33ba6d3290) AuthConfig (ba81a60852c27ccea4253431d16113e65cafe7a82105eda300e687e527122821) AuthConfig (fb50009d4a08af8234509e50b4c769db82bdbbc0d1b4b901294581986412a2a9) AuthConfig (0f81c26485f281d94493156328df483f5df9ff06ff6da868edbf2960d4d29eb5) AuthConfig (ca21a2cc4a2fd8fd9b255e38a644a6e8b735637b43f4dbfbaedfd7c6479cace9) AuthConfig (3d47f76b9a50802ac2c4bf3cc0a600b5f48c41eb9c23e80abfb3588e3782b886) AuthConfig (136ed36efd1b1bc9dbd29e50fd2af5e6432c6458321b3f71c55c9e979ffbb08b) AuthConfig (52293d8507c6ff0e73f3090bd7f0454f93c59f369e141d4ae434aa972a17f0e6) AuthConfig (b347ceebed28c4a95806893c61720f9adaf84254fd57b7af2f4ac9d6f5b63cbc) AuthConfig (989e7bbab87c9c4ee60d599fb0e6830bba2352a499bca89f08be49446ae70b19) AuthConfig (b062bccdbb066072d44d982e027a6c6f9101f59853b2ffc15fb263e749f0ec3f) AuthConfig (c75c5c8e03269df63044da1d9479ecbebb5b36b3b86eae056c259d4e0758928e)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/e2e/tests/multitenancy_helpers.py:269: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:1242: in make_tenant_model_accessible wait_for_status_phase( test/e2e/tests/multitenancy_helpers.py:324: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-90f683-auth' namespace = 'ai-tenant-e2e-inf-a-90f683' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f54b6514ae0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-90f683-auth in ai-tenant-e2e-inf-a-90f683 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-90f683-auth","namespace":"ai-tenant-e2e-inf-a-90f683"},"spec":{"modelRefs":[{"name":"test-model-90f683","namespace":"ai-tenant-e2e-inf-a-90f683"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-29T23:36:17Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-90f683-auth', 'namespace': 'ai-tenant-e2e-inf-a-90f683', 'resourceVersion': '152507', 'uid': '3ba68066-4273-4bbb-8f50-7fa67c7737cb'}, 'spec': {'modelRefs': [{'name': 'test-model-90f683', 'namespace': 'ai-tenant-e2e-inf-a-90f683'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-29T23:36:17Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-90f683-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (61ceb1cbbddc83dc49cabb63dfbe82ce57108c09488c38b284d3a9c12882a181) AuthConfig (e81da58c401266637bb649e74b4d74bbf7a2b4aa75d0277aa0ab96625f9712c5) AuthConfig (a58cb8bcc1268a079aacc0f127a19087d2995c585dca7e812c6eb668b76428f4) AuthConfig (af0b8a3821de4a24044cba30bb35af2b90fea05d98c5fc98b6710881a45c1476) AuthConfig (172db0cf2837c4747bc23ebf9d56e91fa3df7b6a251672bac83b084b19574005) AuthConfig (8a1b50732a941e5c62158e1e23136121dfafdcafd3444ccc3ca4f5e0760858af) AuthConfig (c4b690b8b4b7bf7270db0a65e96ca915e68fc2e2d80a7fb07375cc33ba6d3290) AuthConfig (ba81a60852c27ccea4253431d16113e65cafe7a82105eda300e687e527122821) AuthConfig (fb50009d4a08af8234509e50b4c769db82bdbbc0d1b4b901294581986412a2a9) AuthConfig (0f81c26485f281d94493156328df483f5df9ff06ff6da868edbf2960d4d29eb5) AuthConfig (ca21a2cc4a2fd8fd9b255e38a644a6e8b735637b43f4dbfbaedfd7c6479cace9) AuthConfig (3d47f76b9a50802ac2c4bf3cc0a600b5f48c41eb9c23e80abfb3588e3782b886) AuthConfig (136ed36efd1b1bc9dbd29e50fd2af5e6432c6458321b3f71c55c9e979ffbb08b) AuthConfig (52293d8507c6ff0e73f3090bd7f0454f93c59f369e141d4ae434aa972a17f0e6) AuthConfig (b347ceebed28c4a95806893c61720f9adaf84254fd57b7af2f4ac9d6f5b63cbc) AuthConfig (989e7bbab87c9c4ee60d599fb0e6830bba2352a499bca89f08be49446ae70b19) AuthConfig (b062bccdbb066072d44d982e027a6c6f9101f59853b2ffc15fb263e749f0ec3f) AuthConfig (c75c5c8e03269df63044da1d9479ecbebb5b36b3b86eae056c259d4e0758928e)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/e2e/tests/multitenancy_helpers.py:269: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:1074: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-b-55c3fc', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f1534853740>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-b-55c3fc in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-b-55c3fc","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-b-55c3fc"}}}\n'}, 'creationTimestamp': '2026-10-29T23:40:14Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-b-55c3fc', 'namespace': 'ai-tenants', 'resourceVersion': '156222', 'uid': 'b6ab2f7b-5724-4059-b2c1-063a0ee3e5da'}, 'spec': {'gateway': {'name': 'e2e-inf-b-55c3fc'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-29T23:40:15Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-b-55c3fc', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-b-55c3fc'}} test/e2e/tests/multitenancy_helpers.py:269: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): > bootstrap_aitenant_tenant(case) test/e2e/tests/test_tenant_model_inference.py:153: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:1074: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-inf-b-55c3fc', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f1534853740>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-inf-b-55c3fc in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-inf-b-55c3fc","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-inf-b-55c3fc"}}}\n'}, 'creationTimestamp': '2026-10-29T23:40:14Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-inf-b-55c3fc', 'namespace': 'ai-tenants', 'resourceVersion': '156222', 'uid': 'b6ab2f7b-5724-4059-b2c1-063a0ee3e5da'}, 'spec': {'gateway': {'name': 'e2e-inf-b-55c3fc'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-29T23:40:15Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-inf-b-55c3fc', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-inf-b-55c3fc'}} test/e2e/tests/multitenancy_helpers.py:269: AssertionError