/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_tenant_namespace_discovery.TestTenantNamespaceDiscovery object at 0x7fcce9f2ac90> def test_labeled_tenant_namespace_is_discovered(self): """1.1: Controller discovers labeled tenant namespaces and reconciles CRs.""" case = new_discovery_case(use_default_gateway=True) try: apply_discovery_labels(case["tenant_ns"], case["tenant_label_name"]) apply_tenant_cr(case["tenant_ns"], DEFAULT_GATEWAY_NAME) apply_maas_auth_policy(case["policy_name"], case["tenant_ns"]) apply_maas_subscription(case["subscription_name"], case["tenant_ns"]) wait_for_finalizer("maasauthpolicy", case["policy_name"], case["tenant_ns"], FINALIZER_AUTHPOLICY) wait_for_finalizer("maassubscription", case["subscription_name"], case["tenant_ns"], FINALIZER_SUBSCRIPTION) > auth = wait_for_status_phase( "maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active", ) test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:92: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-policy-b4d9a3d1' namespace = 'ai-tenant-e2e-mt-b4d9a3d1' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fcce9a33060> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-policy-b4d9a3d1 in ai-tenant-e2e-mt-b4d9a3d1 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-policy-b4d9a3d1","namespace":"ai-tenant-e2e-mt-b4d9a3d1"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T17:45:09Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-policy-b4d9a3d1', 'namespace': 'ai-tenant-e2e-mt-b4d9a3d1', 'resourceVersion': '52432', 'uid': '369f5a61-53f5-49b4-9204-e333a6bf4130'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T17:45:10Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 29', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrorself = <test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks object at 0x7f634df49f10> def test_models_as_a_service_namespace_reconciles(self): ns = legacy_default_namespace() suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-default-ns-{suffix}" try: apply_maas_auth_policy(policy_name, ns) wait_for_finalizer("maasauthpolicy", policy_name, ns, FINALIZER_AUTHPOLICY) > wait_for_status_phase("maasauthpolicy", policy_name, ns, expected_phase="Active") test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:394: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-default-ns-e83e5da2' namespace = 'models-as-a-service' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f634dcc1260> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-default-ns-e83e5da2 in models-as-a-service did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-default-ns-e83e5da2","namespace":"models-as-a-service"},"spec":{"modelRefs":[{"name":"facebook-opt-125m-simulated","namespace":"llm"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T17:45:53Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-default-ns-e83e5da2', 'namespace': 'models-as-a-service', 'resourceVersion': '54381', 'uid': 'fb7027f2-a1e1-4640-84ad-5079e108b53a'}, 'spec': {'modelRefs': [{'name': 'facebook-opt-125m-simulated', 'namespace': 'llm'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T17:45:54Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/maas-gateway-auth to be accepted and enforced: AuthPolicy status observed generation 2, current generation is 38', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-d5b905-auth' namespace = 'ai-tenant-e2e-inf-b-d5b905' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f0ce000fd80> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-d5b905-auth in ai-tenant-e2e-inf-b-d5b905 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-d5b905-auth","namespace":"ai-tenant-e2e-inf-b-d5b905"},"spec":{"modelRefs":[{"name":"test-model-d5b905","namespace":"ai-tenant-e2e-inf-b-d5b905"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:12Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-d5b905-auth', 'namespace': 'ai-tenant-e2e-inf-b-d5b905', 'resourceVersion': '92902', 'uid': '875dac32-ab5e-4d83-9071-2d421e64e5dd'}, 'spec': {'modelRefs': [{'name': 'test-model-d5b905', 'namespace': 'ai-tenant-e2e-inf-b-d5b905'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:12Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-b-d5b905-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (90054946b412d9a5438caf17da75c157a0023d02361bd193d12f14d04128e137) AuthConfig (e71ef6dce14bc1880fc8dc805c7c5777a1f9104647ef14bf5b26e628d65799bf) AuthConfig (a7c36b579cbc6f5f7f3167414a773bba4556d9115e3a70e5fb01b634e6528b0f) AuthConfig (760c2e4e97e20ff89daed983874240e10076c6d47e8eb39026c4b9951240dc24) AuthConfig (c01a0279692db076722c7a97236af851947466691081083a345a4dc7a9cd8a13) AuthConfig (f92ae3e587b5f21e8c867744cc89036e9b954bc84f3ad85f68d502ca9ebcf934) AuthConfig (dfdfe021c671820500218705f78356360da3d71c7dbf41ba275506b797e6abc9) AuthConfig (503a49fee2aa0976bdba9bbe2b961deaeac239413ed127c4c7f2ae25c984260d) AuthConfig (54b31c1d15eb2cb6c35449414f94ee2497e8c498bec5d70c14a48c33667a06f2) AuthConfig (f5676996769e69660f916b160074c202b3b6343b49d9bc714d0b205a66016078) AuthConfig (c875289f70e19a306ea46aeeafcca6d9b9c014363b05c849f60b3321a0bfe29a) AuthConfig (e51373f7fc45fbb4f06146022d05574a3d8c72bbeeecf562ef18332617da4a37) AuthConfig (6f43e8120714d44588eefa727269a91eb710cf74acbd013abb9e888adc2c65c9) AuthConfig (9729779596f96bf5df53d3fe66b46a25c4e192ac4f3bbf12a35693f13440ec11) AuthConfig (b3e4d611924a0331e5bb4094b1d778a1461a7838e71772db9ae3e61df12874f0) AuthConfig (8c940ea4e98a78d831a0f752d2775901e4775ebd12446fde907865c1fea5f02b) AuthConfig (93f32aa47ddcec70c791d114cfa26fd547a3c8f9087ba47b1d1f5ab52257d4b0) AuthConfig (99f99256337adb620a66730d60affba61fbf3e3f52b0b33c15125aac5b79f19f)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-d5b905-auth' namespace = 'ai-tenant-e2e-inf-b-d5b905' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f0ce000fd80> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-d5b905-auth in ai-tenant-e2e-inf-b-d5b905 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-d5b905-auth","namespace":"ai-tenant-e2e-inf-b-d5b905"},"spec":{"modelRefs":[{"name":"test-model-d5b905","namespace":"ai-tenant-e2e-inf-b-d5b905"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:12Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-d5b905-auth', 'namespace': 'ai-tenant-e2e-inf-b-d5b905', 'resourceVersion': '92902', 'uid': '875dac32-ab5e-4d83-9071-2d421e64e5dd'}, 'spec': {'modelRefs': [{'name': 'test-model-d5b905', 'namespace': 'ai-tenant-e2e-inf-b-d5b905'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:12Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-b-d5b905-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (90054946b412d9a5438caf17da75c157a0023d02361bd193d12f14d04128e137) AuthConfig (e71ef6dce14bc1880fc8dc805c7c5777a1f9104647ef14bf5b26e628d65799bf) AuthConfig (a7c36b579cbc6f5f7f3167414a773bba4556d9115e3a70e5fb01b634e6528b0f) AuthConfig (760c2e4e97e20ff89daed983874240e10076c6d47e8eb39026c4b9951240dc24) AuthConfig (c01a0279692db076722c7a97236af851947466691081083a345a4dc7a9cd8a13) AuthConfig (f92ae3e587b5f21e8c867744cc89036e9b954bc84f3ad85f68d502ca9ebcf934) AuthConfig (dfdfe021c671820500218705f78356360da3d71c7dbf41ba275506b797e6abc9) AuthConfig (503a49fee2aa0976bdba9bbe2b961deaeac239413ed127c4c7f2ae25c984260d) AuthConfig (54b31c1d15eb2cb6c35449414f94ee2497e8c498bec5d70c14a48c33667a06f2) AuthConfig (f5676996769e69660f916b160074c202b3b6343b49d9bc714d0b205a66016078) AuthConfig (c875289f70e19a306ea46aeeafcca6d9b9c014363b05c849f60b3321a0bfe29a) AuthConfig (e51373f7fc45fbb4f06146022d05574a3d8c72bbeeecf562ef18332617da4a37) AuthConfig (6f43e8120714d44588eefa727269a91eb710cf74acbd013abb9e888adc2c65c9) AuthConfig (9729779596f96bf5df53d3fe66b46a25c4e192ac4f3bbf12a35693f13440ec11) AuthConfig (b3e4d611924a0331e5bb4094b1d778a1461a7838e71772db9ae3e61df12874f0) AuthConfig (8c940ea4e98a78d831a0f752d2775901e4775ebd12446fde907865c1fea5f02b) AuthConfig (93f32aa47ddcec70c791d114cfa26fd547a3c8f9087ba47b1d1f5ab52257d4b0) AuthConfig (99f99256337adb620a66730d60affba61fbf3e3f52b0b33c15125aac5b79f19f)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-2c9b1d-auth' namespace = 'ai-tenant-e2e-inf-b-2c9b1d' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f200f280220> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-2c9b1d-auth in ai-tenant-e2e-inf-b-2c9b1d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-2c9b1d-auth","namespace":"ai-tenant-e2e-inf-b-2c9b1d"},"spec":{"modelRefs":[{"name":"test-model-2c9b1d","namespace":"ai-tenant-e2e-inf-b-2c9b1d"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:11Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-2c9b1d-auth', 'namespace': 'ai-tenant-e2e-inf-b-2c9b1d', 'resourceVersion': '92897', 'uid': 'd5e3c846-b831-419e-becf-c8dcbc2a7238'}, 'spec': {'modelRefs': [{'name': 'test-model-2c9b1d', 'namespace': 'ai-tenant-e2e-inf-b-2c9b1d'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:12Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-b-2c9b1d-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (ec394103ab80a5a0037d11ddbe4489c8edd7a7fe7c0f9f099e9300059cc85cae) AuthConfig (12f96ce8b1f9ce52ee2ef4ed3da67b4fe6d12135f438499cdfca63a685e0118c) AuthConfig (065f845b565d0acac61b120dfcca13ad93b6a82dc5f492403b7cfa75b86f19b5) AuthConfig (b02be290c6507138a34032f25cd10d2ddc735b6977cd87a99efa37bdcc61dbb4) AuthConfig (2583ed3069ea64af538be5da282483087f59bdde7d196771c3736b46056f1184) AuthConfig (467edcaa73c5a1aa9a8160ece9e65f31cec895d1d16aa984d827e6831e6d35ba) AuthConfig (45eb67d75c1d00aa652e594fd30a34d990819fbf963182b97072a74f06b28d7d) AuthConfig (45914e71de64f0b78ccab640767a249750f7ab00d8b2f1d2a6cb700fd24940b4) AuthConfig (f5885909e2e61c1abdd02c3ba3c7e81372b01f8421d3d94ee3a55493f29ce28a) AuthConfig (f43b2718fee3ae1e3098c6e4fb6f75f530b4e6ee9e5b504f1fe457dd41082f5e) AuthConfig (202b21b3a341c6ff122dda11488086fc891fcdbf4774ce773c98be1eafbbeb5e) AuthConfig (54c8812c5de7ac5e98acc523e5371b07eb9ed2dddfd8186a5c6aa2b924eeaa81) AuthConfig (04336c1fbc0fd297688635bc618897811690c7e07e54fbfbd02a79467e25d0dd) AuthConfig (28b144b433d79e6f4bc597e8ef0efc38ae9e36a1d647ad9ac216dd573a73909f) AuthConfig (18165baced909811a537b4a4e3d5d91726652385c69d1a23a4662008e54a9413) AuthConfig (81bc1000a6e7ce7eded4690766e819a2482f7e7265272b3627ce4b1d0911cdc3) AuthConfig (c4aefefc62fff0c1e9ab88627a16133290970f6f91b6139ee2c324c5604a33bb) AuthConfig (318137f1b58057799bd10eb5d09aa96e6d8a2bc0c6132ef66425143b1aa18886)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-2c9b1d-auth' namespace = 'ai-tenant-e2e-inf-b-2c9b1d' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f200f280220> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-2c9b1d-auth in ai-tenant-e2e-inf-b-2c9b1d did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-2c9b1d-auth","namespace":"ai-tenant-e2e-inf-b-2c9b1d"},"spec":{"modelRefs":[{"name":"test-model-2c9b1d","namespace":"ai-tenant-e2e-inf-b-2c9b1d"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:11Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-2c9b1d-auth', 'namespace': 'ai-tenant-e2e-inf-b-2c9b1d', 'resourceVersion': '92897', 'uid': 'd5e3c846-b831-419e-becf-c8dcbc2a7238'}, 'spec': {'modelRefs': [{'name': 'test-model-2c9b1d', 'namespace': 'ai-tenant-e2e-inf-b-2c9b1d'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:12Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-b-2c9b1d-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (ec394103ab80a5a0037d11ddbe4489c8edd7a7fe7c0f9f099e9300059cc85cae) AuthConfig (12f96ce8b1f9ce52ee2ef4ed3da67b4fe6d12135f438499cdfca63a685e0118c) AuthConfig (065f845b565d0acac61b120dfcca13ad93b6a82dc5f492403b7cfa75b86f19b5) AuthConfig (b02be290c6507138a34032f25cd10d2ddc735b6977cd87a99efa37bdcc61dbb4) AuthConfig (2583ed3069ea64af538be5da282483087f59bdde7d196771c3736b46056f1184) AuthConfig (467edcaa73c5a1aa9a8160ece9e65f31cec895d1d16aa984d827e6831e6d35ba) AuthConfig (45eb67d75c1d00aa652e594fd30a34d990819fbf963182b97072a74f06b28d7d) AuthConfig (45914e71de64f0b78ccab640767a249750f7ab00d8b2f1d2a6cb700fd24940b4) AuthConfig (f5885909e2e61c1abdd02c3ba3c7e81372b01f8421d3d94ee3a55493f29ce28a) AuthConfig (f43b2718fee3ae1e3098c6e4fb6f75f530b4e6ee9e5b504f1fe457dd41082f5e) AuthConfig (202b21b3a341c6ff122dda11488086fc891fcdbf4774ce773c98be1eafbbeb5e) AuthConfig (54c8812c5de7ac5e98acc523e5371b07eb9ed2dddfd8186a5c6aa2b924eeaa81) AuthConfig (04336c1fbc0fd297688635bc618897811690c7e07e54fbfbd02a79467e25d0dd) AuthConfig (28b144b433d79e6f4bc597e8ef0efc38ae9e36a1d647ad9ac216dd573a73909f) AuthConfig (18165baced909811a537b4a4e3d5d91726652385c69d1a23a4662008e54a9413) AuthConfig (81bc1000a6e7ce7eded4690766e819a2482f7e7265272b3627ce4b1d0911cdc3) AuthConfig (c4aefefc62fff0c1e9ab88627a16133290970f6f91b6139ee2c324c5604a33bb) AuthConfig (318137f1b58057799bd10eb5d09aa96e6d8a2bc0c6132ef66425143b1aa18886)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-96768b-auth' namespace = 'ai-tenant-e2e-inf-a-96768b' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f0de5ca8f40> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-96768b-auth in ai-tenant-e2e-inf-a-96768b did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-96768b-auth","namespace":"ai-tenant-e2e-inf-a-96768b"},"spec":{"modelRefs":[{"name":"test-model-96768b","namespace":"ai-tenant-e2e-inf-a-96768b"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:23Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-96768b-auth', 'namespace': 'ai-tenant-e2e-inf-a-96768b', 'resourceVersion': '92670', 'uid': '2d38c13b-3031-4024-a992-a581a5bbdee2'}, 'spec': {'modelRefs': [{'name': 'test-model-96768b', 'namespace': 'ai-tenant-e2e-inf-a-96768b'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:23Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-96768b-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (fa46732204eb89640e5c7dc9f1fec0565445a1590770bdb5b9680ad460524601) AuthConfig (c8b401de911d1dbbb8e0f0a90215153564d1a8bdc262854e7a5ebc47661a86a2) AuthConfig (62c0c13d1acbab70e7123ffe848f0ddefe132695700d42a9314893b0806882ac) AuthConfig (ae2e4fc6474bb4cdb2954fb3af943b971272614f69b9c0fb93e802b42a1fbb02) AuthConfig (44daef693b53a5e2d0bd9fe555d68d88986cab14406cc6e26d3c53047fbef9e3) AuthConfig (d54feed3057080898d119a577ef323c16cbf984a8d48363a6b2a2b0db7f7f7c0) AuthConfig (06c3827491d62b5ad14c6002a1b1df526875259613f16c1425a0ef56772ff451) AuthConfig (903c48c42a8a8e738fafa39ae4474b38457c6d2d40c855adef2003ac222a5495) AuthConfig (d032b4be85c14bd85b3cdb48ae0b60ed624ab87f3dfb887cafa6b4e58196b81f) AuthConfig (efe93d371e218cebff84335881df373d04a2d8d08e4d0b77d51610929e9491ec) AuthConfig (6a6a6f09b986a76d785507410d89290d2fc0f387cc21316d73657bc37f124d09) AuthConfig (00af1f4ba52fe85885f63c85db849f5287cad03e9086980b1b6d5008d4fd5eba) AuthConfig (ae431b91451acc0975df21ea35721fdc359577f76c3b80aba4353224aadaf42e) AuthConfig (a9f2cc1d28d9d483cc4a0aa0e2f266d4319367e86c4b8c82ced21a5d831d2e59) AuthConfig (a4be2e5aea88ac1bc94927e9a7f22d9d4f7233fe0d0a099fc8173b7d2458b09a) AuthConfig (e7430dafbb4f44cb2bf341e6991d8820e5c03f464735c0154346432a8430fe99) AuthConfig (5512ae15c1f111aa854920b49de312a06d12c6af90a50fd4f4dff40fde09bb68) AuthConfig (2476976496d1732c32843f25f37b78cfb2158d2fafede889f853fe71893d55c4)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-96768b-auth' namespace = 'ai-tenant-e2e-inf-a-96768b' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f0de5ca8f40> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-96768b-auth in ai-tenant-e2e-inf-a-96768b did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-96768b-auth","namespace":"ai-tenant-e2e-inf-a-96768b"},"spec":{"modelRefs":[{"name":"test-model-96768b","namespace":"ai-tenant-e2e-inf-a-96768b"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:23Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-96768b-auth', 'namespace': 'ai-tenant-e2e-inf-a-96768b', 'resourceVersion': '92670', 'uid': '2d38c13b-3031-4024-a992-a581a5bbdee2'}, 'spec': {'modelRefs': [{'name': 'test-model-96768b', 'namespace': 'ai-tenant-e2e-inf-a-96768b'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:23Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-96768b-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (fa46732204eb89640e5c7dc9f1fec0565445a1590770bdb5b9680ad460524601) AuthConfig (c8b401de911d1dbbb8e0f0a90215153564d1a8bdc262854e7a5ebc47661a86a2) AuthConfig (62c0c13d1acbab70e7123ffe848f0ddefe132695700d42a9314893b0806882ac) AuthConfig (ae2e4fc6474bb4cdb2954fb3af943b971272614f69b9c0fb93e802b42a1fbb02) AuthConfig (44daef693b53a5e2d0bd9fe555d68d88986cab14406cc6e26d3c53047fbef9e3) AuthConfig (d54feed3057080898d119a577ef323c16cbf984a8d48363a6b2a2b0db7f7f7c0) AuthConfig (06c3827491d62b5ad14c6002a1b1df526875259613f16c1425a0ef56772ff451) AuthConfig (903c48c42a8a8e738fafa39ae4474b38457c6d2d40c855adef2003ac222a5495) AuthConfig (d032b4be85c14bd85b3cdb48ae0b60ed624ab87f3dfb887cafa6b4e58196b81f) AuthConfig (efe93d371e218cebff84335881df373d04a2d8d08e4d0b77d51610929e9491ec) AuthConfig (6a6a6f09b986a76d785507410d89290d2fc0f387cc21316d73657bc37f124d09) AuthConfig (00af1f4ba52fe85885f63c85db849f5287cad03e9086980b1b6d5008d4fd5eba) AuthConfig (ae431b91451acc0975df21ea35721fdc359577f76c3b80aba4353224aadaf42e) AuthConfig (a9f2cc1d28d9d483cc4a0aa0e2f266d4319367e86c4b8c82ced21a5d831d2e59) AuthConfig (a4be2e5aea88ac1bc94927e9a7f22d9d4f7233fe0d0a099fc8173b7d2458b09a) AuthConfig (e7430dafbb4f44cb2bf341e6991d8820e5c03f464735c0154346432a8430fe99) AuthConfig (5512ae15c1f111aa854920b49de312a06d12c6af90a50fd4f4dff40fde09bb68) AuthConfig (2476976496d1732c32843f25f37b78cfb2158d2fafede889f853fe71893d55c4)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrortenant_env = ({'name': 'e2e-shared-a-w3-557feb', 'namespace': 'ai-tenant-e2e-shared-a-w3-557feb', 'base_url': 'https://e2e-shared-a...-b-w3-227939.apps.de06ce2496430a335bdc.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-227939', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-664e79' namespace = 'ai-tenant-e2e-shared-a-w3-557feb' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f1a728f8040> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-664e79 in ai-tenant-e2e-shared-a-w3-557feb did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-664e79","namespace":"ai-tenant-e2e-shared-a-w3-557feb"},"spec":{"modelRefs":[{"name":"auth-test-model-557feb","namespace":"ai-tenant-e2e-shared-a-w3-557feb"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:08:31Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-664e79', 'namespace': 'ai-tenant-e2e-shared-a-w3-557feb', 'resourceVersion': '104000', 'uid': '80fc9104-8519-43be-8f08-a455c1a7f9b9'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-557feb', 'namespace': 'ai-tenant-e2e-shared-a-w3-557feb'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:08:38Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w3-557feb-maas-auth to be accepted and enforced: AuthPolicy status has not reported an observed generation', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionErrortenant_env = ({'name': 'e2e-shared-a-w3-557feb', 'namespace': 'ai-tenant-e2e-shared-a-w3-557feb', 'base_url': 'https://e2e-shared-a...-b-w3-227939.apps.de06ce2496430a335bdc.prod.konflux-ocp-ci.dev/maas-api', 'model_name': 'auth-test-model-227939', ...}) @pytest.fixture(scope="module") def tenant_auth_setup(tenant_env): tenant_a, tenant_b = tenant_env suffix = uuid.uuid4().hex[:6] policy_name = f"e2e-auth-iso-{suffix}" subscription_name = f"e2e-auth-iso-{suffix}" try: for tenant in tenant_env: > make_tenant_model_accessible( tenant["model_name"], tenant["namespace"], policy_name, subscription_name, gateway_name=tenant["gateway_name"], trlp_timeout=MODEL_BACKEND_READY_TIMEOUT, ) test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:135: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-auth-iso-664e79' namespace = 'ai-tenant-e2e-shared-a-w3-557feb' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f1a728f8040> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/e2e-auth-iso-664e79 in ai-tenant-e2e-shared-a-w3-557feb did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"e2e-auth-iso-664e79","namespace":"ai-tenant-e2e-shared-a-w3-557feb"},"spec":{"modelRefs":[{"name":"auth-test-model-557feb","namespace":"ai-tenant-e2e-shared-a-w3-557feb"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:08:31Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'e2e-auth-iso-664e79', 'namespace': 'ai-tenant-e2e-shared-a-w3-557feb', 'resourceVersion': '104000', 'uid': '80fc9104-8519-43be-8f08-a455c1a7f9b9'}, 'spec': {'modelRefs': [{'name': 'auth-test-model-557feb', 'namespace': 'ai-tenant-e2e-shared-a-w3-557feb'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:08:38Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-shared-a-w3-557feb-maas-auth to be accepted and enforced: AuthPolicy status has not reported an observed generation', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-872354-auth' namespace = 'ai-tenant-e2e-inf-a-872354' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fcce9fe3880> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-872354-auth in ai-tenant-e2e-inf-a-872354 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-872354-auth","namespace":"ai-tenant-e2e-inf-a-872354"},"spec":{"modelRefs":[{"name":"test-model-872354","namespace":"ai-tenant-e2e-inf-a-872354"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:23Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-872354-auth', 'namespace': 'ai-tenant-e2e-inf-a-872354', 'resourceVersion': '92711', 'uid': '467b0a83-b7b4-42b5-a801-972c372d4f81'}, 'spec': {'modelRefs': [{'name': 'test-model-872354', 'namespace': 'ai-tenant-e2e-inf-a-872354'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:23Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-872354-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (7886e3b70984a327ce3c2e75e14c26c9b318451460b371aea2d23dffdad2f652) AuthConfig (6b3d8e93e66331cc0670ea6b7d060ec60b0bbed480b475adcd732ccc78f2d5a4) AuthConfig (4882d787c925be03f76af45e1dea8c13cc171e11073055da1aa486afd51e075e) AuthConfig (025d6477efa378b1a78b341bbce364d4a44de02ecd31c693723a724a51007caa) AuthConfig (6993f8c3ad62c42ae9ed35147c9b8296ed8ee926a6a50ca742e3ea256a422f33) AuthConfig (bc6e243a94b5be0674559773dee64d0a442aa716d1a9c7a8bda24435fbc73774) AuthConfig (449bfb7baf30e0985931cad678fb99b62037fbae5423fc59493b7e0ee64e5df7) AuthConfig (d3822a0eeefd0bff71e886a7ea7ed52077d9bafeba14879642e45c3a6d4ac374) AuthConfig (0defa0869115927c6a338042083c7fe094034505350c1e81a3a4526bc05971ff) AuthConfig (c9676dc1aca4c197d656ba41aff943a6536bf420137b8dae67cd35a833862d1d) AuthConfig (946004b250c5671ccf5688af46a207990a3905e7606e29ab566c9346a43fbe22) AuthConfig (6432a4927c44f58f2c5c2733eabbee92c4109d9c5dcbda1ece6d7c21bc20ffaf) AuthConfig (8f7c8655638844821a6f3e853cc78d7c814d82b06f001c306e1c8a866dc39c85) AuthConfig (ce23aed649198acc5a3fa1cf83a13bd80992df6824ea2704b50a6c0ebd3fcd91) AuthConfig (798fdded8c2049173ff1d933adc86e557e03d9bb6988eb86840936f0fe139572) AuthConfig (e87e5af0e4cfb3a496493cafd1bb94d52db02a40820562843d68c29bb67fbac7) AuthConfig (392374c08929ea4a4f58bb6f5e9176a082289036ba884ae9805828343d515f45) AuthConfig (05566fad0be375c73cd3f7c3395f4d8755830a0f2dc533a089b8e5d3abbf8f3d)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-872354-auth' namespace = 'ai-tenant-e2e-inf-a-872354' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7fcce9fe3880> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-872354-auth in ai-tenant-e2e-inf-a-872354 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-872354-auth","namespace":"ai-tenant-e2e-inf-a-872354"},"spec":{"modelRefs":[{"name":"test-model-872354","namespace":"ai-tenant-e2e-inf-a-872354"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:23Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-872354-auth', 'namespace': 'ai-tenant-e2e-inf-a-872354', 'resourceVersion': '92711', 'uid': '467b0a83-b7b4-42b5-a801-972c372d4f81'}, 'spec': {'modelRefs': [{'name': 'test-model-872354', 'namespace': 'ai-tenant-e2e-inf-a-872354'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:23Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-a-872354-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (7886e3b70984a327ce3c2e75e14c26c9b318451460b371aea2d23dffdad2f652) AuthConfig (6b3d8e93e66331cc0670ea6b7d060ec60b0bbed480b475adcd732ccc78f2d5a4) AuthConfig (4882d787c925be03f76af45e1dea8c13cc171e11073055da1aa486afd51e075e) AuthConfig (025d6477efa378b1a78b341bbce364d4a44de02ecd31c693723a724a51007caa) AuthConfig (6993f8c3ad62c42ae9ed35147c9b8296ed8ee926a6a50ca742e3ea256a422f33) AuthConfig (bc6e243a94b5be0674559773dee64d0a442aa716d1a9c7a8bda24435fbc73774) AuthConfig (449bfb7baf30e0985931cad678fb99b62037fbae5423fc59493b7e0ee64e5df7) AuthConfig (d3822a0eeefd0bff71e886a7ea7ed52077d9bafeba14879642e45c3a6d4ac374) AuthConfig (0defa0869115927c6a338042083c7fe094034505350c1e81a3a4526bc05971ff) AuthConfig (c9676dc1aca4c197d656ba41aff943a6536bf420137b8dae67cd35a833862d1d) AuthConfig (946004b250c5671ccf5688af46a207990a3905e7606e29ab566c9346a43fbe22) AuthConfig (6432a4927c44f58f2c5c2733eabbee92c4109d9c5dcbda1ece6d7c21bc20ffaf) AuthConfig (8f7c8655638844821a6f3e853cc78d7c814d82b06f001c306e1c8a866dc39c85) AuthConfig (ce23aed649198acc5a3fa1cf83a13bd80992df6824ea2704b50a6c0ebd3fcd91) AuthConfig (798fdded8c2049173ff1d933adc86e557e03d9bb6988eb86840936f0fe139572) AuthConfig (e87e5af0e4cfb3a496493cafd1bb94d52db02a40820562843d68c29bb67fbac7) AuthConfig (392374c08929ea4a4f58bb6f5e9176a082289036ba884ae9805828343d515f45) AuthConfig (05566fad0be375c73cd3f7c3395f4d8755830a0f2dc533a089b8e5d3abbf8f3d)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-b2fc9f-auth' namespace = 'ai-tenant-e2e-inf-b-b2fc9f' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f0c1372f4c0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-b2fc9f-auth in ai-tenant-e2e-inf-b-b2fc9f did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-b2fc9f-auth","namespace":"ai-tenant-e2e-inf-b-b2fc9f"},"spec":{"modelRefs":[{"name":"test-model-b2fc9f","namespace":"ai-tenant-e2e-inf-b-b2fc9f"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:10Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-b2fc9f-auth', 'namespace': 'ai-tenant-e2e-inf-b-b2fc9f', 'resourceVersion': '92883', 'uid': 'ac66a8f8-0196-4078-9fe8-34171799ecc7'}, 'spec': {'modelRefs': [{'name': 'test-model-b2fc9f', 'namespace': 'ai-tenant-e2e-inf-b-b2fc9f'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:10Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-b-b2fc9f-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (402a613e0006994f2ca8d1e136b2395ecd6b553fd4d50352429c2f64ceb178aa) AuthConfig (a246796480d71ca9280e6ed6d8b23cbefaa85aa02c14f913cf3d3b9c2e745096) AuthConfig (382887bdb062f7d303180b65568a123d07435d2f8318cc785dc401be686706bb) AuthConfig (518c90fdbb471b9ea541b1b1ef39d1fccefe00a34df78f93e96982508c9ca59a) AuthConfig (7cc875c29c6cc2a3eccf8226e9fabc353871efccde2ef98e44a27db3875f907c) AuthConfig (162051dc09da331a9cd57c7f960eea4eb54575c1342d2700070289fe724c2690) AuthConfig (9a87692e97f3ceb1165dbb798a9f53ffd33e9dddfc25974a5efd5c0081f218b0) AuthConfig (308c81e0a4d6bfe471d3081cb355709487e4b398d1ac040dc12d0d545671456f) AuthConfig (a06576a82b59267222f836284414bce881db54c5f7a5e9e9e6c056ce657955d7) AuthConfig (a78179e19701d8fa2d02503f47f00d32e6ec5941f1ebc60f0702ee2c9f3b3e52) AuthConfig (d35e37e4462c94fc1be8b3118eeecc2e03c6a7e165790ce74024e6830bb75206) AuthConfig (f4a2496537ff7e2dc77011934240f96f738ade42715d3e519792caefa381f82d) AuthConfig (a9fbc4dcc07ca58fd7e44a5837d999819948c69cbb9d79c5c5bcaf8d430356aa) AuthConfig (a025980ecdc0721611a617814287cb6d4f8548c5e7ad14160d9d6f20cfc1febf) AuthConfig (eeeaeca82b78dde5cc1170e0a48c29c2ae4cbfa6e1f3e7329046b32884720bc5) AuthConfig (ad64732bdf528a1157ed015da900b6f643030a510094c8c8f35349c3db3696c0) AuthConfig (9ec2d4ce41fc818592256e17641a0d2ab01bf44fc781901d02af3c1fc4e0d0a1) AuthConfig (b6e9e43534714fc724cd2c46c19cd0ab1e4f1c5237db64cac8b5c2a4da2cceda)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) > make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:177: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1296: in make_tenant_model_accessible wait_for_status_phase( test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:343: in wait_for_status_phase return wait_for_json(kind, name, namespace, predicate=_predicate, timeout=timeout, interval=interval) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'test-model-b2fc9f-auth' namespace = 'ai-tenant-e2e-inf-b-b2fc9f' predicate = <function wait_for_status_phase.<locals>._predicate at 0x7f0c1372f4c0> timeout = 180, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: maasauthpolicy/test-model-b2fc9f-auth in ai-tenant-e2e-inf-b-b2fc9f did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"MaaSAuthPolicy","metadata":{"annotations":{},"name":"test-model-b2fc9f-auth","namespace":"ai-tenant-e2e-inf-b-b2fc9f"},"spec":{"modelRefs":[{"name":"test-model-b2fc9f","namespace":"ai-tenant-e2e-inf-b-b2fc9f"}],"subjects":{"groups":[{"name":"system:authenticated"}]}}}\n'}, 'creationTimestamp': '2026-11-02T18:02:10Z', 'finalizers': ['maas.opendatahub.io/authpolicy-cleanup'], 'generation': 1, 'name': 'test-model-b2fc9f-auth', 'namespace': 'ai-tenant-e2e-inf-b-b2fc9f', 'resourceVersion': '92883', 'uid': 'ac66a8f8-0196-4078-9fe8-34171799ecc7'}, 'spec': {'modelRefs': [{'name': 'test-model-b2fc9f', 'namespace': 'ai-tenant-e2e-inf-b-b2fc9f'}], 'subjects': {'groups': [{'name': 'system:authenticated'}]}}, 'status': {'conditions': [{'lastTransitionTime': '2026-11-02T18:02:10Z', 'message': 'Waiting for gateway AuthPolicy openshift-ingress/e2e-inf-b-b2fc9f-maas-auth to be accepted and enforced: AuthPolicy waiting for the following components to sync: [AuthConfig (402a613e0006994f2ca8d1e136b2395ecd6b553fd4d50352429c2f64ceb178aa) AuthConfig (a246796480d71ca9280e6ed6d8b23cbefaa85aa02c14f913cf3d3b9c2e745096) AuthConfig (382887bdb062f7d303180b65568a123d07435d2f8318cc785dc401be686706bb) AuthConfig (518c90fdbb471b9ea541b1b1ef39d1fccefe00a34df78f93e96982508c9ca59a) AuthConfig (7cc875c29c6cc2a3eccf8226e9fabc353871efccde2ef98e44a27db3875f907c) AuthConfig (162051dc09da331a9cd57c7f960eea4eb54575c1342d2700070289fe724c2690) AuthConfig (9a87692e97f3ceb1165dbb798a9f53ffd33e9dddfc25974a5efd5c0081f218b0) AuthConfig (308c81e0a4d6bfe471d3081cb355709487e4b398d1ac040dc12d0d545671456f) AuthConfig (a06576a82b59267222f836284414bce881db54c5f7a5e9e9e6c056ce657955d7) AuthConfig (a78179e19701d8fa2d02503f47f00d32e6ec5941f1ebc60f0702ee2c9f3b3e52) AuthConfig (d35e37e4462c94fc1be8b3118eeecc2e03c6a7e165790ce74024e6830bb75206) AuthConfig (f4a2496537ff7e2dc77011934240f96f738ade42715d3e519792caefa381f82d) AuthConfig (a9fbc4dcc07ca58fd7e44a5837d999819948c69cbb9d79c5c5bcaf8d430356aa) AuthConfig (a025980ecdc0721611a617814287cb6d4f8548c5e7ad14160d9d6f20cfc1febf) AuthConfig (eeeaeca82b78dde5cc1170e0a48c29c2ae4cbfa6e1f3e7329046b32884720bc5) AuthConfig (ad64732bdf528a1157ed015da900b6f643030a510094c8c8f35349c3db3696c0) AuthConfig (9ec2d4ce41fc818592256e17641a0d2ab01bf44fc781901d02af3c1fc4e0d0a1) AuthConfig (b6e9e43534714fc724cd2c46c19cd0ab1e4f1c5237db64cac8b5c2a4da2cceda)]', 'observedGeneration': 1, 'reason': 'Unknown', 'status': 'Unknown', 'type': 'Ready'}], 'phase': 'Pending'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:288: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) # Store model path (with /v1 for OpenAI API compatibility) case["model_path"] = f"/{case['tenant_ns']}/{model_name}/v1" for case in (case_a, case_b): > wait_for_route_auth_enforced( f"{_get_tenant_gateway_url(case['gateway_name'])}{case['model_path']}" ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:189: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:430: in wait_for_route_auth_enforced wait_until( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ check = <function wait_for_route_auth_enforced.<locals>._check at 0x7f634dcc18a0> timeout = 180 what = 'https://e2e-inf-b-8a209f.apps.de06ce2496430a335bdc.prod.konflux-ocp-ci.dev/ai-tenant-e2e-inf-b-8a209f/test-model-8a209f/v1/chat/completions did not reject an unknown API key for 10s' interval = 2 def wait_until(check, timeout: int, what: str, interval: int = 5): """Poll check() until it returns a truthy value. Assertion failures and transient cluster or HTTP errors count as "not yet"; the last one ends up in the timeout message.""" deadline = time.time() + timeout while True: try: result = check() if result: return result last = result except (AssertionError, subprocess.TimeoutExpired, requests.RequestException) as exc: last = exc > assert time.time() < deadline, f"{what} within {timeout}s; last: {last}" ^^^^^^^^^^^^^^^^^^^^^^ E AssertionError: https://e2e-inf-b-8a209f.apps.de06ce2496430a335bdc.prod.konflux-ocp-ci.dev/ai-tenant-e2e-inf-b-8a209f/test-model-8a209f/v1/chat/completions did not reject an unknown API key for 10s within 180s; last: status=500 x-ext-auth-reason=None body=Internal Server Error. test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:266: AssertionError@pytest.fixture(scope="module") def tenant_inference_cases(): """Set up two tenants with models for inference testing.""" require_aitenant_crd() case_a = new_named_tenant_case("e2e-inf-a") case_b = new_named_tenant_case("e2e-inf-b") try: # Bootstrap tenants for case in (case_a, case_b): bootstrap_aitenant_tenant(case) # Create both LLMInferenceServices before waiting on either. The llmisvc # controller reconciles them one at a time and runs minutes behind under # parallel load, so B should queue while A's wait runs, not after it. for case in (case_a, case_b): # Track model name early for cleanup case["model_name"] = f"test-model-{case['suffix']}" _create_llmis( case["model_name"], case["tenant_ns"], case["gateway_name"], GATEWAY_NAMESPACE, ) for case in (case_a, case_b): model_name = case["model_name"] # Re-applies the LLMIS created above unchanged, then waits for it. provision_tenant_model( model_name, case["tenant_ns"], case["gateway_name"], ready_timeout=MODEL_BACKEND_READY_TIMEOUT, ) make_tenant_model_accessible( model_name, case["tenant_ns"], f"{model_name}-auth", f"{model_name}-sub", token_limit=10000, gateway_name=case["gateway_name"], ) # Store model path (with /v1 for OpenAI API compatibility) case["model_path"] = f"/{case['tenant_ns']}/{model_name}/v1" for case in (case_a, case_b): > wait_for_route_auth_enforced( f"{_get_tenant_gateway_url(case['gateway_name'])}{case['model_path']}" ) test/maas-e2e/test/e2e/tests/test_tenant_model_inference.py:189: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:430: in wait_for_route_auth_enforced wait_until( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ check = <function wait_for_route_auth_enforced.<locals>._check at 0x7f634dcc18a0> timeout = 180 what = 'https://e2e-inf-b-8a209f.apps.de06ce2496430a335bdc.prod.konflux-ocp-ci.dev/ai-tenant-e2e-inf-b-8a209f/test-model-8a209f/v1/chat/completions did not reject an unknown API key for 10s' interval = 2 def wait_until(check, timeout: int, what: str, interval: int = 5): """Poll check() until it returns a truthy value. Assertion failures and transient cluster or HTTP errors count as "not yet"; the last one ends up in the timeout message.""" deadline = time.time() + timeout while True: try: result = check() if result: return result last = result except (AssertionError, subprocess.TimeoutExpired, requests.RequestException) as exc: last = exc > assert time.time() < deadline, f"{what} within {timeout}s; last: {last}" ^^^^^^^^^^^^^^^^^^^^^^ E AssertionError: https://e2e-inf-b-8a209f.apps.de06ce2496430a335bdc.prod.konflux-ocp-ci.dev/ai-tenant-e2e-inf-b-8a209f/test-model-8a209f/v1/chat/completions did not reject an unknown API key for 10s within 180s; last: status=500 x-ext-auth-reason=None body=Internal Server Error. test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:266: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B