/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7f29d5c12f50> def test_labeled_tenant_namespace_is_discovered(self): """1.1: Controller discovers labeled tenant namespaces and reconciles CRs.""" case = new_discovery_case(use_default_gateway=True) try: apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", case["subscription_name"], case["tenant_ns"], FINALIZER_SUBSCRIPTION) > auth = wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-34ccb1f0' namespace = 'ai-tenant-e2e-mt-34ccb1f0' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f29d5883ec0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-34ccb1f0 in ai-tenant-e2e-mt-34ccb1f0 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-34ccb1f0","namespace":"ai-tenant-e2e-mt-34ccb1f0"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:31:31Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-34ccb1f0', 'namespace': 'ai-tenant-e2e-mt-34ccb1f0', 'resourceVersion': '59310', 'uid': '36ca978d-516d-4883-a1d6-a8aeffdcf36e'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:34:25Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 37', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks object at 0x7ff60ef46290> def test_models_as_a_service_namespace_reconciles(self): ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-default-ns-{suffix}" try: apply_maas_auth_policy(policy_name, ns) wait_for_finalizer("maasauthpolicy", policy_name, ns, FINALIZER_AUTHPOLICY) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:394: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-ns-2b2e4151' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7ff60ec19620> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-ns-2b2e4151 in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-ns-2b2e4151","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:32:18Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-ns-2b2e4151', 'namespace': 'models-as-a-service', 'resourceVersion': '61767', 'uid': '6cb57f28-717a-4092-89c7-e3254df89355'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:32:18Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 56', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorself = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7f229a5b4a90> def test_dynamic_discovery_after_label_added(self): """1.2 variant: Pre-existing CRs reconcile after namespace gains discovery labels.""" case = new_discovery_case(use_default_gateway=True) try: ensure_namespace(case["tenant_ns"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) time.sleep(10) before = get_json_or_none("maasauthpolicy", case["policy_name"], case["tenant_ns"]) assert before is not None assert FINALIZER_AUTHPOLICY not in ((before.get("metadata") or {}).get("finalizers") or []) apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) > wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-32f20bfa' namespace = 'ai-tenant-e2e-mt-32f20bfa' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f229a1fee80> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-32f20bfa in ai-tenant-e2e-mt-32f20bfa did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-32f20bfa","namespace":"ai-tenant-e2e-mt-32f20bfa"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:32:23Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-32f20bfa', 'namespace': 'ai-tenant-e2e-mt-32f20bfa', 'resourceVersion': '62995', 'uid': '0f153b70-f563-4f6a-b772-c943f3e0bd5b'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:32:33Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 63', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_model_inference.TestTenantModelInference object at 0x7fa12c7bc650> tenant_inference_cases = ({'suffix': 'a5d9b6', 'tenant_ns': 'ai-tenant-e2e-inf-a-a5d9b6', 'tenant_label_name': 'e2e-inf-a-a5d9b6', 'gateway_nam...t_ns': 'ai-tenant-e2e-inf-b-86a40d', 'tenant_label_name': 'e2e-inf-b-86a40d', 'gateway_name': 'e2e-inf-b-86a40d', ...}) def test_tenant_isolation_cross_gateway_blocked(self, tenant_inference_cases): """Tenant isolation: tenant A cannot access tenant B's model via B's gateway.""" case_a, case_b = tenant_inference_cases # Get gateway URLs gateway_a_url = _get_tenant_gateway_url(case_a["gateway_name"]) gateway_b_url = _get_tenant_gateway_url(case_b["gateway_name"]) api_key_a = _create_tenant_api_key(gateway_a_url, case_a, name_prefix="e2e-cross-tenant-a") api_key_b = _create_tenant_api_key(gateway_b_url, case_b, name_prefix="e2e-cross-tenant-b") # Both positive controls must pass before a cross-tenant denial can be # attributed to the target gateway's tenant-specific auth configuration. _wait_for_chat_ok(case_a, gateway_a_url, api_key_a) _wait_for_chat_ok(case_b, gateway_b_url, api_key_b) for source_case, target_case, target_gateway_url, foreign_key in ( (case_a, case_b, gateway_b_url, api_key_a), (case_b, case_a, gateway_a_url, api_key_b), ): model_url = f"{target_gateway_url}{target_case['model_path']}" with _tenant_gateway_auth_policy(target_case["gateway_name"]): response = _request_with_gateway_retry( requests.post, f"{model_url}/chat/completions", headers={"Authorization": f"Bearer {foreign_key}"}, json={ "model": "facebook/opt-125m", "messages": [{"role": "user", "content": "Say hello"}], }, ) > assert is_gateway_auth_denial(response), ( f"Expected {target_case['tenant_ns']} auth to reject " f"{source_case['tenant_ns']}'s key with 401/403, got {response.status_code}. " f"Response: {response_summary(response, max_body=500)}" ) E AssertionError: Expected ai-tenant-e2e-inf-a-a5d9b6 auth to reject ai-tenant-e2e-inf-b-86a40d's key with 401/403, got 500. Response: status=500 x-ext-auth-reason=None body=Internal Server Error. E E assert False E + where False = is_gateway_auth_denial(<Response [500]>) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:312: AssertionErrorself = <test_tenant_model_inference.TestTenantGatewayNegativeSecurity object at 0x7fa12c54b950> tenant_inference_cases = ({'suffix': 'a5d9b6', 'tenant_ns': 'ai-tenant-e2e-inf-a-a5d9b6', 'tenant_label_name': 'e2e-inf-a-a5d9b6', 'gateway_nam...t_ns': 'ai-tenant-e2e-inf-b-86a40d', 'tenant_label_name': 'e2e-inf-b-86a40d', 'gateway_name': 'e2e-inf-b-86a40d', ...}) tenant_index = 1 @pytest.mark.parametrize("tenant_index", [0, 1], ids=["tenant_a", "tenant_b"]) def test_forged_identity_headers_denied_on_tenant_gateway(self, tenant_inference_cases, tenant_index): """Client-supplied X-MaaS identity headers are denied on each tenant gateway (RHOAIENG-99369).""" case = tenant_inference_cases[tenant_index] gateway_url = _get_tenant_gateway_url(case["gateway_name"]) > api_key = _create_tenant_api_key(gateway_url, case, name_prefix="e2e-tenant-neg-sec") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:388: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ gateway_url = 'https://e2e-inf-b-86a40d.apps.ff566a2b099b3f36c4dd.prod.konflux-ocp-ci.dev' case = {'suffix': '86a40d', 'tenant_ns': 'ai-tenant-e2e-inf-b-86a40d', 'tenant_label_name': 'e2e-inf-b-86a40d', 'gateway_name': 'e2e-inf-b-86a40d', ...} name_prefix = 'e2e-tenant-neg-sec' def _create_tenant_api_key(gateway_url, case, *, name_prefix="e2e-body-routing"): """Create an API key for a tenant and return it.""" oc_token = _get_cluster_token() r = create_api_key_at( f"{gateway_url}/maas-api", oc_token, f"{name_prefix}-{case['suffix']}", subscription=f"{case['model_name']}-sub", ) > assert r.status_code in (200, 201), ( f"Failed to create tenant API key: {r.status_code} {redact_sensitive(r.text)}" ) E AssertionError: Failed to create tenant API key: 504 <html><body><h1>504 Gateway Time-out</h1> E The server didn't respond in time. E </body></html> E E assert 504 in (200, 201) E + where 504 = <Response [504]>.status_code test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:125: AssertionErrortenant_env = ({'name': 'e2e-shared-a-w0-b8fd22', 'namespace': 'ai-tenant-e2e-shared-a-w0-b8fd22', 'base_url': 'https://e2e-shared-a...-b-w0-63258b.apps.ff566a2b099b3f36c4dd.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-63258b', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-9ab273' namespace = 'ai-tenant-e2e-shared-a-w0-b8fd22' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f9a048d4a40> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-9ab273 in ai-tenant-e2e-shared-a-w0-b8fd22 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-9ab273","namespace":"ai-tenant-e2e-shared-a-w0-b8fd22"},"spec":{"modelRefs":[{"name":"auth-test-model-b8fd22","namespace":"ai-tenant-e2e-shared-a-w0-b8fd22"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:49:52Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-9ab273', 'namespace': 'ai-tenant-e2e-shared-a-w0-b8fd22', 'resourceVersion': '117352', 'uid': 'be94cc0c-aec7-4ac5-81e8-4588d47c6e2d'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-b8fd22', 'namespace': 'ai-tenant-e2e-shared-a-w0-b8fd22'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:49:53Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w0-b8fd22-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (c4e2fabec3d05e089298473c5952403449320ee65b18831445b9dbf61c3db61c) AuthConfig (4e69b8eb639ff715f7de161b59da5ba489d756b7bf713b296ac6adba80e40ad3) AuthConfig (a8087ba4ffe2bda8562ab43525e237d974be2098a367b45eb8c1ce313bf4437b) AuthConfig (b6190cd27117e814d8200604ed4ad41d026350ecbb940c763c62904afd7dc396) AuthConfig (8a5bc29553ef4384a74b98e58a8192284d3c19ebac8b41bebfd156e3f9d80e97) AuthConfig (07a50e548a3cba9ff37fec4f374cdee05424bd943f9f31d16a50dea6f6afedd9) AuthConfig (f26889287d2f1d3ace915bade179697b14d401ba1b10edce0ad89459a9d01f9d) AuthConfig (8ece2166d04fcd7e77286b7bed69cc11888f62eb8ad6561f1e16229cc93073f8) AuthConfig (7f72264340ebd4e83437a9a799b850d13b39d7e0bc4ca10dd8f6d2859c7b258d) AuthConfig (226b950a6bfe3e5f756b4a6c89dcb2cb6a9f2ccddf92067699f08903e14e4337) AuthConfig (3fe9f0e4d0a0d95e5b345ef88bf69e50bf099388a60b46395bbc5b5003a167c0) AuthConfig (fa9c0984799359946ef53bdebd3685443a580337a1d751e24f71877823bffa5d) AuthConfig (73aef5003c14e18953387bcfbbd85b5b277953444f665701795f8168f79472be) AuthConfig (6e48d8440953a7deb3ade5c8d27a2a60f72535ed3c8be5d5c5198d36bd8eba2f) AuthConfig (c221ccc2df2e816bb5c60ebc19709c7057bb7ed12ffec14613718d4cb4307853) AuthConfig (2f3a91c4623c9c3b10972d9b9a81118cbb4821d6323ac2b4a2bc4656757b6901) AuthConfig (1c8e0821b0dcfc1e31bea70590f262fdd93312aaf13100a57419e79033d4e42d) AuthConfig (44584b3dc715b28b6fe8e35c24f551c9304c60b55bac094182616e4b04a7c4d8)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_Btenant_env = ({'name': 'e2e-shared-a-w4-d4145b', 'namespace': 'ai-tenant-e2e-shared-a-w4-d4145b', 'base_url': 'https://e2e-shared-a...-b-w4-86a834.apps.ff566a2b099b3f36c4dd.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-86a834', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-f495fc' namespace = 'ai-tenant-e2e-shared-a-w4-d4145b' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7ff60f15ec00> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-f495fc in ai-tenant-e2e-shared-a-w4-d4145b did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-f495fc","namespace":"ai-tenant-e2e-shared-a-w4-d4145b"},"spec":{"modelRefs":[{"name":"auth-test-model-d4145b","namespace":"ai-tenant-e2e-shared-a-w4-d4145b"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:49:49Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-f495fc', 'namespace': 'ai-tenant-e2e-shared-a-w4-d4145b', 'resourceVersion': '117428', 'uid': '40592593-3382-4f47-8a78-ae89c6b95dc8'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-d4145b', 'namespace': 'ai-tenant-e2e-shared-a-w4-d4145b'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:49:49Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w4-d4145b-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (2a089204a568ef6a875882fddbe63e24c4c7481e055118416989dca7f6bfe2f0) AuthConfig (357e9d90d5f180954f616c68695eae237d1c249e665abdf79430cf14095d2712) AuthConfig (3c82c7f152d198dcc7610a7837b5d1556b4a32105b1338f4ecc434876d14d8e6) AuthConfig (f87117e6e7415a50976164e3249ac2a466ad3807f646150149a3ac014f52e83a) AuthConfig (ad538531b195bd5c2573ed228ed99e3eef9742fe54bfb49e53c26a0543d925c1) AuthConfig (3166259981ba17c8b2cc404bb0e2dbd8b0cce2b698465726f671f7877e571f31) AuthConfig (829facbf0db5abbc6fa07d4b442a140842b643f93fa709cbec7e46e9c4947393) AuthConfig (8c5049c470ab048443c7bb58f1e63f39e3ed47c4818882350fcbc0e91ea1eb72) AuthConfig (98f41f452fed63d375a9377d1c062542853a96a6238ca7a594304622570d3e0b) AuthConfig (a52d98d5e3ae8278a29c85436e2ba02416f46085a3b1595ddffbf09e19d21b7a) AuthConfig (efbbba8373c25b37c1a7e71e2133a3dac651cef53779fea1c28d820da2618900) AuthConfig (4792d3ff694b12c52cf22c48f993383e84f8cfed805f37b8255943879e90eba6) AuthConfig (8598bbc5ad47604daeb289ba524b6dbce1782dde4cbb97d4a82a864e98dee405) AuthConfig (1e3ce7a12e4fffec94801f25729db58715076c4e8b5758f4d3f5b864540590b8) AuthConfig (185ac63b854377c5803aafcb304e28f0cde11fdd06788cf634cf4d5db2c9dc90) AuthConfig (839034788fd2489f9450a59f8b29efa1a149e8a7685b75a3cac65a2cb099bda5) AuthConfig (b40cf61f7241b5595ee79953c39228711b72e4d66af40b5536056fda44d87e52) AuthConfig (4b9eb47ed4d0df0565b1f65477b8e42364c9565311b57ab81e5f0c13816f7e3c)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrortenant_env = ({'name': 'e2e-shared-a-w4-d4145b', 'namespace': 'ai-tenant-e2e-shared-a-w4-d4145b', 'base_url': 'https://e2e-shared-a...-b-w4-86a834.apps.ff566a2b099b3f36c4dd.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-86a834', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-f495fc' namespace = 'ai-tenant-e2e-shared-a-w4-d4145b' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7ff60f15ec00> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-f495fc in ai-tenant-e2e-shared-a-w4-d4145b did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-f495fc","namespace":"ai-tenant-e2e-shared-a-w4-d4145b"},"spec":{"modelRefs":[{"name":"auth-test-model-d4145b","namespace":"ai-tenant-e2e-shared-a-w4-d4145b"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:49:49Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-f495fc', 'namespace': 'ai-tenant-e2e-shared-a-w4-d4145b', 'resourceVersion': '117428', 'uid': '40592593-3382-4f47-8a78-ae89c6b95dc8'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-d4145b', 'namespace': 'ai-tenant-e2e-shared-a-w4-d4145b'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:49:49Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w4-d4145b-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (2a089204a568ef6a875882fddbe63e24c4c7481e055118416989dca7f6bfe2f0) AuthConfig (357e9d90d5f180954f616c68695eae237d1c249e665abdf79430cf14095d2712) AuthConfig (3c82c7f152d198dcc7610a7837b5d1556b4a32105b1338f4ecc434876d14d8e6) AuthConfig (f87117e6e7415a50976164e3249ac2a466ad3807f646150149a3ac014f52e83a) AuthConfig (ad538531b195bd5c2573ed228ed99e3eef9742fe54bfb49e53c26a0543d925c1) AuthConfig (3166259981ba17c8b2cc404bb0e2dbd8b0cce2b698465726f671f7877e571f31) AuthConfig (829facbf0db5abbc6fa07d4b442a140842b643f93fa709cbec7e46e9c4947393) AuthConfig (8c5049c470ab048443c7bb58f1e63f39e3ed47c4818882350fcbc0e91ea1eb72) AuthConfig (98f41f452fed63d375a9377d1c062542853a96a6238ca7a594304622570d3e0b) AuthConfig (a52d98d5e3ae8278a29c85436e2ba02416f46085a3b1595ddffbf09e19d21b7a) AuthConfig (efbbba8373c25b37c1a7e71e2133a3dac651cef53779fea1c28d820da2618900) AuthConfig (4792d3ff694b12c52cf22c48f993383e84f8cfed805f37b8255943879e90eba6) AuthConfig (8598bbc5ad47604daeb289ba524b6dbce1782dde4cbb97d4a82a864e98dee405) AuthConfig (1e3ce7a12e4fffec94801f25729db58715076c4e8b5758f4d3f5b864540590b8) AuthConfig (185ac63b854377c5803aafcb304e28f0cde11fdd06788cf634cf4d5db2c9dc90) AuthConfig (839034788fd2489f9450a59f8b29efa1a149e8a7685b75a3cac65a2cb099bda5) AuthConfig (b40cf61f7241b5595ee79953c39228711b72e4d66af40b5536056fda44d87e52) AuthConfig (4b9eb47ed4d0df0565b1f65477b8e42364c9565311b57ab81e5f0c13816f7e3c)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrortenant_env = ({'name': 'e2e-shared-a-w3-afb8be', 'namespace': 'ai-tenant-e2e-shared-a-w3-afb8be', 'base_url': 'https://e2e-shared-a...-b-w3-d04aad.apps.ff566a2b099b3f36c4dd.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-d04aad', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-68807e' namespace = 'ai-tenant-e2e-shared-a-w3-afb8be' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f229a797ec0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-68807e in ai-tenant-e2e-shared-a-w3-afb8be did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-68807e","namespace":"ai-tenant-e2e-shared-a-w3-afb8be"},"spec":{"modelRefs":[{"name":"auth-test-model-afb8be","namespace":"ai-tenant-e2e-shared-a-w3-afb8be"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:49:37Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-68807e', 'namespace': 'ai-tenant-e2e-shared-a-w3-afb8be', 'resourceVersion': '117301', 'uid': 'd00b01fe-e02e-48ce-8b7c-cc4536d71ac8'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-afb8be', 'namespace': 'ai-tenant-e2e-shared-a-w3-afb8be'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:49:37Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w3-afb8be-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (5788ade88a60f2a76ad026622a4e985ecbd368190f0388ccb206f2f21cdabf39) AuthConfig (696bdd3f72b9742e1d45cab967ae5a6c9d57c5f11781cc505c21f32f69442a40) AuthConfig (8a9d4809c80c4a024715bc96d14df2d7d8ebcf5ad8dcb009f6089c7517836d8f) AuthConfig (bfdaa824ab6a09f15eda2ce3baa51b535633e0ea46dcda923561f7a3dd409737) AuthConfig (ddba1aba0da8e64a8f701706f2fb7157bb23037a4b61242df4e521dad49045cd) AuthConfig (910f0df7207b881e98bbf854a1c8e8b098d3591bffed829d05cd886f9516a89e) AuthConfig (d37736659632a9a4a944cb29aa11919d77c6a3d5c12b1768a3ec29ee9f9b10c8) AuthConfig (94c87cc50978a4964124f47df1011b23812071b49d13a0a88b53f0c62b592d9e) AuthConfig (d5565532e2cf6eef8d7bbbb023572f32160d3d9524bbffe54ca13d85bdb2bbf3) AuthConfig (3a548aad43828f3da39da20acb71114523c24bc4f07cff680c2740d4bc7732ee) AuthConfig (a48f5cc1a4a11afbca9f856cbed2790a056f2beacf866ad2080eb00d66e4c150) AuthConfig (42f7fedb1372b893bec895041281ec916d33cb2374ef01ca792dbe3e2840b6ca) AuthConfig (b625c0184369903d877e94fcfe2c46513efe0575bc0effc183a7950af4455b2b) AuthConfig (ad23f1051203585dbc1e95c8bb45255d51ae62e49f3be8d55033237aca8b55ec) AuthConfig (9cdb1edd26a7c0a4b1a629a848427e4c91c82ee2d0276aac8268cae583665cd1) AuthConfig (fa52921d1952b95276a4a89a88dbd31c98535e9ee249815210e2ea74dcbb4b35) AuthConfig (55224013db13e89c5cdaea1d2e4e515a96cfd943faf17cb87c2fc3a3a469bde8) AuthConfig (63665d63eeb6ddc102420acdb92fbbd19c443c0f823cfbc2c22ad647cf868857)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrortenant_env = ({'name': 'e2e-shared-a-w3-afb8be', 'namespace': 'ai-tenant-e2e-shared-a-w3-afb8be', 'base_url': 'https://e2e-shared-a...-b-w3-d04aad.apps.ff566a2b099b3f36c4dd.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-d04aad', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-68807e' namespace = 'ai-tenant-e2e-shared-a-w3-afb8be' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f229a797ec0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-68807e in ai-tenant-e2e-shared-a-w3-afb8be did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-68807e","namespace":"ai-tenant-e2e-shared-a-w3-afb8be"},"spec":{"modelRefs":[{"name":"auth-test-model-afb8be","namespace":"ai-tenant-e2e-shared-a-w3-afb8be"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-10-31T03:49:37Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-68807e', 'namespace': 'ai-tenant-e2e-shared-a-w3-afb8be', 'resourceVersion': '117301', 'uid': 'd00b01fe-e02e-48ce-8b7c-cc4536d71ac8'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-afb8be', 'namespace': 'ai-tenant-e2e-shared-a-w3-afb8be'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-31T03:49:37Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w3-afb8be-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (5788ade88a60f2a76ad026622a4e985ecbd368190f0388ccb206f2f21cdabf39) AuthConfig (696bdd3f72b9742e1d45cab967ae5a6c9d57c5f11781cc505c21f32f69442a40) AuthConfig (8a9d4809c80c4a024715bc96d14df2d7d8ebcf5ad8dcb009f6089c7517836d8f) AuthConfig (bfdaa824ab6a09f15eda2ce3baa51b535633e0ea46dcda923561f7a3dd409737) AuthConfig (ddba1aba0da8e64a8f701706f2fb7157bb23037a4b61242df4e521dad49045cd) AuthConfig (910f0df7207b881e98bbf854a1c8e8b098d3591bffed829d05cd886f9516a89e) AuthConfig (d37736659632a9a4a944cb29aa11919d77c6a3d5c12b1768a3ec29ee9f9b10c8) AuthConfig (94c87cc50978a4964124f47df1011b23812071b49d13a0a88b53f0c62b592d9e) AuthConfig (d5565532e2cf6eef8d7bbbb023572f32160d3d9524bbffe54ca13d85bdb2bbf3) AuthConfig (3a548aad43828f3da39da20acb71114523c24bc4f07cff680c2740d4bc7732ee) AuthConfig (a48f5cc1a4a11afbca9f856cbed2790a056f2beacf866ad2080eb00d66e4c150) AuthConfig (42f7fedb1372b893bec895041281ec916d33cb2374ef01ca792dbe3e2840b6ca) AuthConfig (b625c0184369903d877e94fcfe2c46513efe0575bc0effc183a7950af4455b2b) AuthConfig (ad23f1051203585dbc1e95c8bb45255d51ae62e49f3be8d55033237aca8b55ec) AuthConfig (9cdb1edd26a7c0a4b1a629a848427e4c91c82ee2d0276aac8268cae583665cd1) AuthConfig (fa52921d1952b95276a4a89a88dbd31c98535e9ee249815210e2ea74dcbb4b35) AuthConfig (55224013db13e89c5cdaea1d2e4e515a96cfd943faf17cb87c2fc3a3a469bde8) AuthConfig (63665d63eeb6ddc102420acdb92fbbd19c443c0f823cfbc2c22ad647cf868857)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1320: in make_tenant_model_accessible _wait_for_subscription_trlp_status( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'test-model-b68780-sub', expected_ready = True namespace = 'ai-tenant-e2e-inf-b-b68780', timeout = 180, model = None def _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=None, timeout=SUBSCRIPTION_TRLP_STATUS_TIMEOUT, model=None, ): """Wait for MaaSSubscription's TokenRateLimitPolicy status to reach expected ready state. Args: name: Name of the MaaSSubscription expected_ready: Expected ready state for all TRLPs (True or False) namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) model: Optional model name. When set, only that model's mirrored TRLP status is considered. Returns: The subscription CR dict when all TRLPs reach the expected ready state Raises: TimeoutError: If TRLPs don't reach expected state within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} TRLP ready={expected_ready} (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] # If we expect ready and there are no TRLPs yet, keep waiting if expected_ready and len(trlp_statuses) == 0: log.debug(f"MaaSSubscription {name}: waiting for TRLP statuses to appear") time.sleep(2) continue # Check if all TRLPs match expected ready state if len(trlp_statuses) > 0: all_match = all(trlp.get("ready") == expected_ready for trlp in trlp_statuses) if all_match: log.info(f"✅ MaaSSubscription {name} has {len(trlp_statuses)} TRLP(s) with ready={expected_ready}") return cr log.debug(f"MaaSSubscription {name}: TRLP statuses={trlp_statuses}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] > raise TimeoutError( f"MaaSSubscription {namespace}/{name} TRLPs" f"{f' for model {model}' if model else ''} did not reach ready={expected_ready} " f"within {timeout}s (current TRLPs: {trlp_statuses})" ) E TimeoutError: MaaSSubscription ai-tenant-e2e-inf-b-b68780/test-model-b68780-sub TRLPs did not reach ready=True within 180s (current TRLPs: [{'message': 'status not available on gateway openshift-ingress/e2e-inf-b-b68780', 'model': 'test-model-b68780', 'name': 'maas-trlp-test-model-b68780', 'namespace': 'ai-tenant-e2e-inf-b-b68780', 'ready': False, 'reason': 'NotAccepted'}]) test/maas-e2e/test/e2e/tests/test_helper.py:1796: TimeoutError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1320: in make_tenant_model_accessible _wait_for_subscription_trlp_status( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'test-model-b68780-sub', expected_ready = True namespace = 'ai-tenant-e2e-inf-b-b68780', timeout = 180, model = None def _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=None, timeout=SUBSCRIPTION_TRLP_STATUS_TIMEOUT, model=None, ): """Wait for MaaSSubscription's TokenRateLimitPolicy status to reach expected ready state. Args: name: Name of the MaaSSubscription expected_ready: Expected ready state for all TRLPs (True or False) namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) model: Optional model name. When set, only that model's mirrored TRLP status is considered. Returns: The subscription CR dict when all TRLPs reach the expected ready state Raises: TimeoutError: If TRLPs don't reach expected state within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} TRLP ready={expected_ready} (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] # If we expect ready and there are no TRLPs yet, keep waiting if expected_ready and len(trlp_statuses) == 0: log.debug(f"MaaSSubscription {name}: waiting for TRLP statuses to appear") time.sleep(2) continue # Check if all TRLPs match expected ready state if len(trlp_statuses) > 0: all_match = all(trlp.get("ready") == expected_ready for trlp in trlp_statuses) if all_match: log.info(f"✅ MaaSSubscription {name} has {len(trlp_statuses)} TRLP(s) with ready={expected_ready}") return cr log.debug(f"MaaSSubscription {name}: TRLP statuses={trlp_statuses}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] > raise TimeoutError( f"MaaSSubscription {namespace}/{name} TRLPs" f"{f' for model {model}' if model else ''} did not reach ready={expected_ready} " f"within {timeout}s (current TRLPs: {trlp_statuses})" ) E TimeoutError: MaaSSubscription ai-tenant-e2e-inf-b-b68780/test-model-b68780-sub TRLPs did not reach ready=True within 180s (current TRLPs: [{'message': 'status not available on gateway openshift-ingress/e2e-inf-b-b68780', 'model': 'test-model-b68780', 'name': 'maas-trlp-test-model-b68780', 'namespace': 'ai-tenant-e2e-inf-b-b68780', 'ready': False, 'reason': 'NotAccepted'}]) test/maas-e2e/test/e2e/tests/test_helper.py:1796: TimeoutError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1320: in make_tenant_model_accessible _wait_for_subscription_trlp_status( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'test-model-b81d65-sub', expected_ready = True namespace = 'ai-tenant-e2e-inf-a-b81d65', timeout = 180, model = None def _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=None, timeout=SUBSCRIPTION_TRLP_STATUS_TIMEOUT, model=None, ): """Wait for MaaSSubscription's TokenRateLimitPolicy status to reach expected ready state. Args: name: Name of the MaaSSubscription expected_ready: Expected ready state for all TRLPs (True or False) namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) model: Optional model name. When set, only that model's mirrored TRLP status is considered. Returns: The subscription CR dict when all TRLPs reach the expected ready state Raises: TimeoutError: If TRLPs don't reach expected state within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} TRLP ready={expected_ready} (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] # If we expect ready and there are no TRLPs yet, keep waiting if expected_ready and len(trlp_statuses) == 0: log.debug(f"MaaSSubscription {name}: waiting for TRLP statuses to appear") time.sleep(2) continue # Check if all TRLPs match expected ready state if len(trlp_statuses) > 0: all_match = all(trlp.get("ready") == expected_ready for trlp in trlp_statuses) if all_match: log.info(f"✅ MaaSSubscription {name} has {len(trlp_statuses)} TRLP(s) with ready={expected_ready}") return cr log.debug(f"MaaSSubscription {name}: TRLP statuses={trlp_statuses}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] > raise TimeoutError( f"MaaSSubscription {namespace}/{name} TRLPs" f"{f' for model {model}' if model else ''} did not reach ready={expected_ready} " f"within {timeout}s (current TRLPs: {trlp_statuses})" ) E TimeoutError: MaaSSubscription ai-tenant-e2e-inf-a-b81d65/test-model-b81d65-sub TRLPs did not reach ready=True within 180s (current TRLPs: [{'message': 'status not available on gateway openshift-ingress/e2e-inf-a-b81d65', 'model': 'test-model-b81d65', 'name': 'maas-trlp-test-model-b81d65', 'namespace': 'ai-tenant-e2e-inf-a-b81d65', 'ready': False, 'reason': 'NotAccepted'}]) test/maas-e2e/test/e2e/tests/test_helper.py:1796: TimeoutError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1320: in make_tenant_model_accessible _wait_for_subscription_trlp_status( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'test-model-b81d65-sub', expected_ready = True namespace = 'ai-tenant-e2e-inf-a-b81d65', timeout = 180, model = None def _wait_for_subscription_trlp_status( name, expected_ready=True, namespace=None, timeout=SUBSCRIPTION_TRLP_STATUS_TIMEOUT, model=None, ): """Wait for MaaSSubscription's TokenRateLimitPolicy status to reach expected ready state. Args: name: Name of the MaaSSubscription expected_ready: Expected ready state for all TRLPs (True or False) namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) model: Optional model name. When set, only that model's mirrored TRLP status is considered. Returns: The subscription CR dict when all TRLPs reach the expected ready state Raises: TimeoutError: If TRLPs don't reach expected state within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSSubscription {name} TRLP ready={expected_ready} (timeout: {timeout}s)...") while time.time() < deadline: cr = _get_cr("maassubscription", name, namespace) if cr: status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] # If we expect ready and there are no TRLPs yet, keep waiting if expected_ready and len(trlp_statuses) == 0: log.debug(f"MaaSSubscription {name}: waiting for TRLP statuses to appear") time.sleep(2) continue # Check if all TRLPs match expected ready state if len(trlp_statuses) > 0: all_match = all(trlp.get("ready") == expected_ready for trlp in trlp_statuses) if all_match: log.info(f"✅ MaaSSubscription {name} has {len(trlp_statuses)} TRLP(s) with ready={expected_ready}") return cr log.debug(f"MaaSSubscription {name}: TRLP statuses={trlp_statuses}") time.sleep(2) # Timeout - return current state for debugging cr = _get_cr("maassubscription", name, namespace) status = cr.get("status", {}) if cr else {} trlp_statuses = status.get("tokenRateLimitStatuses", []) if model is not None: trlp_statuses = [ trlp for trlp in trlp_statuses if trlp.get("model") == model ] > raise TimeoutError( f"MaaSSubscription {namespace}/{name} TRLPs" f"{f' for model {model}' if model else ''} did not reach ready={expected_ready} " f"within {timeout}s (current TRLPs: {trlp_statuses})" ) E TimeoutError: MaaSSubscription ai-tenant-e2e-inf-a-b81d65/test-model-b81d65-sub TRLPs did not reach ready=True within 180s (current TRLPs: [{'message': 'status not available on gateway openshift-ingress/e2e-inf-a-b81d65', 'model': 'test-model-b81d65', 'name': 'maas-trlp-test-model-b81d65', 'namespace': 'ai-tenant-e2e-inf-a-b81d65', 'ready': False, 'reason': 'NotAccepted'}]) test/maas-e2e/test/e2e/tests/test_helper.py:1796: TimeoutError