/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1580: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestE2ESubscriptionFlow object at 0x7f8b00ac38d0> def test_e2e_multiple_subscriptions_separate_keys_gets_200(self): """ User with two subscriptions for the same model: mint one API key per subscription; each key succeeds on inference without x-maas-subscription. """ ns = _ns() auth_policy_name = "e2e-test-auth-multi-sub-valid" subscription_1 = "e2e-test-subscription-free" subscription_2 = "e2e-test-subscription-premium" sa_name = "e2e-sa-multi-sub-valid" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources with 2 subscriptions for the same user _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_1, MODEL_REF, users=[sa_user], token_limit=100) _create_test_subscription(subscription_2, MODEL_REF, users=[sa_user], token_limit=1000) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_1, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2089: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w5-7a3603' model_namespace = 'e2e-models-e2e-worker-w5-7a3603', timeout = 180 subscription_name = 'e2e-test-subscription-free' subscription_namespace = 'ai-tenant-e2e-worker-w5-7a3603' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w5-7a3603 was not created and enforced in e2e-models-e2e-worker-w5-7a3603 within 180s: generation=5, observedGeneration=5, Enforced={'lastTransitionTime': '2026-10-30T13:00:23Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7f5a8698f110> def test_e2e_with_both_access_and_subscription_gets_200(self): """ API key with both access and subscription should get 200 OK. Uses an existing model reference that aligns with gateway path-based model identity. """ ns = _ns() auth_policy_name = "e2e-test-auth-success" subscription_name = "e2e-test-subscription-success" sa_name = "e2e-sa-success" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Use pre-existing premium model to keep model identity consistent with gateway path. _create_test_auth_policy(auth_policy_name, PREMIUM_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, PREMIUM_MODEL_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth_policy_name, namespace=ns, timeout=120, require_auth_policies=False) > _wait_for_subscription_inference_ready( subscription_name, PREMIUM_MODEL_REF, namespace=ns, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1887: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'premium-w3-09db16' model_namespace = 'e2e-models-e2e-worker-w3-09db16', timeout = 180 subscription_name = 'e2e-test-subscription-success' subscription_namespace = 'ai-tenant-e2e-worker-w3-09db16' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-premium-w3-09db16 was not created and enforced in e2e-models-e2e-worker-w3-09db16 within 180s: generation=4, observedGeneration=4, Enforced={'lastTransitionTime': '2026-10-30T13:00:32Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f980f1b4bd0> def test_multiple_api_keys_different_subscriptions(self): """ Test: Multiple API keys each bound to different subscriptions. Creates two API keys from the same user, each explicitly bound to a different subscription. Verifies each key returns only its bound subscription's models. Expected: Each API key returns models only from its bound subscription. """ sa_name = "e2e-multi-keys-sa" sa_ns = "default" maas_ns = _ns() sub1_name = "e2e-multi-keys-sub1" sub2_name = "e2e-multi-keys-sub2" auth1_name = "e2e-multi-keys-auth1" auth2_name = "e2e-multi-keys-auth2" api_key1 = None api_key2 = None try: # Create SA sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create two subscriptions with different models log.info(f"Creating subscription 1 with {DISTINCT_MODEL_REF}") _create_test_auth_policy(auth1_name, DISTINCT_MODEL_REF, users=[sa_user]) _create_test_subscription(sub1_name, DISTINCT_MODEL_REF, users=[sa_user]) log.info(f"Creating subscription 2 with {DISTINCT_MODEL_2_REF}") _create_test_auth_policy(auth2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) _create_test_subscription(sub2_name, DISTINCT_MODEL_2_REF, users=[sa_user]) # Wait for both subscriptions to reconcile before creating API keys _wait_for_subscription_discovery_ready(sub1_name, namespace=maas_ns) _wait_for_subscription_discovery_ready(sub2_name, namespace=maas_ns) # Create two API keys, each bound to a different subscription log.info(f"Creating API key 1 bound to {sub1_name}") api_key1_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key1", "subscription": sub1_name}, ) assert api_key1_response.status_code in (200, 201) api_key1 = api_key1_response.json().get("key") bound_sub1 = api_key1_response.json().get("subscription") assert bound_sub1 == sub1_name, f"Key 1 should be bound to {sub1_name}, got {bound_sub1}" log.info(f"Creating API key 2 bound to {sub2_name}") api_key2_response = _request_with_gateway_retry( requests.post, f"{_maas_api_url()}/v1/api-keys", headers={"Authorization": f"Bearer {sa_token}", "Content-Type": "application/json"}, json={"name": "key2", "subscription": sub2_name}, ) assert api_key2_response.status_code in (200, 201) api_key2 = api_key2_response.json().get("key") bound_sub2 = api_key2_response.json().get("subscription") assert bound_sub2 == sub2_name, f"Key 2 should be bound to {sub2_name}, got {bound_sub2}" _wait_for_maas_auth_policy_phase(auth1_name, require_enforced=False) _wait_for_maas_auth_policy_phase(auth2_name, require_enforced=False) # Both models must be listable before either key's listing can show # it leaves out the other subscription's model. _wait_for_central_models_in_subscription(api_key1, sub1_name) _wait_for_central_models_in_subscription(api_key2, sub2_name) # Test key1 - should return models from sub1 only log.info(f"Testing API key 1 (bound to {sub1_name})") r1 = _get_models_with_gateway_retry( headers={"Authorization": f"Bearer {api_key1}"}, ) assert r1.status_code == 200, f"Expected 200 for key1, got {r1.status_code}: {r1.text}" models1 = r1.json().get("data") or [] model_ids1 = {m["id"] for m in models1} assert DISTINCT_MODEL_ID in model_ids1, f"Key1 should see {DISTINCT_MODEL_ID} from {sub1_name}" assert DISTINCT_MODEL_2_ID not in model_ids1, f"Key1 should NOT see {DISTINCT_MODEL_2_ID} from {sub2_name}" # Test key2 - should return models from sub2 only log.info(f"Testing API key 2 (bound to {sub2_name})") r2 = _get_models_with_gateway_retry( headers={"Authorization": f"Bearer {api_key2}"}, ) assert r2.status_code == 200, f"Expected 200 for key2, got {r2.status_code}: {r2.text}" models2 = r2.json().get("data") or [] model_ids2 = {m["id"] for m in models2} > assert DISTINCT_MODEL_2_ID in model_ids2, f"Key2 should see {DISTINCT_MODEL_2_ID} from {sub2_name}" E AssertionError: Key2 should see publishers/e2e-models-e2e-worker-w0-c90b9e/models/e2e/distinct-2-w0-c90b9e from e2e-multi-keys-sub2 E assert 'publishers/e2e-models-e2e-worker-w0-c90b9e/models/e2e/distinct-2-w0-c90b9e' in set() test/maas-e2e/test/e2e/tests/test_models_endpoint.py:2036: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deployself = <test_subscription.TestE2ESubscriptionFlow object at 0x7f9651ea23d0> def test_e2e_group_based_access_gets_200(self): """ E2E test: Group-based auth and subscription (success case). Validates that users can access models via group membership in both MaaSAuthPolicy and MaaSSubscription, not just explicit user lists. """ ns = _ns() auth_policy_name = "e2e-test-group-auth" subscription_name = "e2e-test-group-subscription" sa_name = "e2e-sa-group" # Use namespace-specific group that SA will be in test_group = f"system:serviceaccounts:{ns}" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) # Create auth policy using GROUP (not user) _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group]) # Create subscription using GROUP (not user) _create_test_subscription(subscription_name, MODEL_REF, groups=[test_group]) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_name, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2216: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w2-c2ce75' model_namespace = 'e2e-models-e2e-worker-w2-c2ce75', timeout = 180 subscription_name = 'e2e-test-group-subscription' subscription_namespace = 'ai-tenant-e2e-worker-w2-c2ce75' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w2-c2ce75 was not created and enforced in e2e-models-e2e-worker-w2-c2ce75 within 180s: generation=4, observedGeneration=4, Enforced={'lastTransitionTime': '2026-10-30T13:06:25Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B@pytest.fixture(scope="module") def ipp_tenant_cases(): require_tenant_namespace_discovery() require_aitenant_crd() case_a = new_named_tenant_case("e2e-ipp-a") case_b = new_named_tenant_case("e2e-ipp-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP) test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ipp-a-78543b', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f980f88aa20>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-ipp-a-78543b in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-a-78543b","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-a-78543b"}}}\n'}, 'creationTimestamp': '2026-10-30T13:33:41Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-a-78543b', 'namespace': 'ai-tenants', 'resourceVersion': '115174', 'uid': 'a3822916-4cff-4af6-94b0-e889a470676d'}, 'spec': {'gateway': {'name': 'e2e-ipp-a-78543b'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T13:33:43Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-a-78543b', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-a-78543b'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError@pytest.fixture(scope="module") def ipp_tenant_cases(): require_tenant_namespace_discovery() require_aitenant_crd() case_a = new_named_tenant_case("e2e-ipp-a") case_b = new_named_tenant_case("e2e-ipp-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP) test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ipp-a-3b47e7', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f9651efef20>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-ipp-a-3b47e7 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-a-3b47e7","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-a-3b47e7"}}}\n'}, 'creationTimestamp': '2026-10-30T13:33:55Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-a-3b47e7', 'namespace': 'ai-tenants', 'resourceVersion': '115768', 'uid': 'cf0fb2b9-641a-455b-9dac-069b06ad900a'}, 'spec': {'gateway': {'name': 'e2e-ipp-a-3b47e7'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T13:33:56Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-a-3b47e7', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-a-3b47e7'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError@pytest.fixture(scope="module") def ipp_tenant_cases(): require_tenant_namespace_discovery() require_aitenant_crd() case_a = new_named_tenant_case("e2e-ipp-a") case_b = new_named_tenant_case("e2e-ipp-b") try: for case in (case_a, case_b): bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP) > wait_for_per_tenant_ipp_ready(case) test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:125: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1336: in wait_for_per_tenant_ipp_ready wait_for_deployment_available(names["processing_deployment"], GATEWAY_NAMESPACE, timeout=timeout) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:428: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'payload-processing-e2e-ipp-b-c91909' namespace = 'openshift-ingress' predicate = <function wait_for_deployment_available.<locals>._predicate at 0x7f48be72ea20> timeout = 240, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/payload-processing-e2e-ipp-b-c91909 in openshift-ingress did not satisfy condition. Last object: {'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': {'annotations': {'deployment.kubernetes.io/revision': '1'}, 'creationTimestamp': '2026-10-30T13:34:34Z', 'generation': 1, 'labels': {'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service', 'app.opendatahub.io/modelsasservice': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-ipp-b-c91909', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-ipp-b-c91909'}, 'name': 'payload-processing-e2e-ipp-b-c91909', 'namespace': 'openshift-ingress', 'ownerReferences': [{'apiVersion': 'maas.opendatahub.io/v1alpha1', 'blockOwnerDeletion': True, 'controller': True, 'kind': 'Config', 'name': 'default', 'uid': 'b7a60aef-fc0c-4831-96cb-da865b6e4ee8'}], 'resourceVersion': '116914', 'uid': 'c8aea13d-acdc-406d-998a-5c513168ac24'}, 'spec': {'progressDeadlineSeconds': 600, 'replicas': 1, 'revisionHistoryLimit': 10, 'selector': {'matchLabels': {'app': 'payload-processing', 'maas.opendatahub.io/tenant-instance': 'payload-processing-e2e-ipp-b-c91909'}}, 'strategy': {'rollingUpdate': {'maxSurge': '25%', 'maxUnavailable': '25%'}, 'type': 'RollingUpdate'}, 'template': {'metadata': {'labels': {'app': 'payload-processing', 'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service', 'maas.opendatahub.io/tenant-instance': 'payload-processing-e2e-ipp-b-c91909'}}, 'spec': {'containers': [{'args': ['--config-file', '/config/custom-ipp-config.yaml', '--v=3', '--tracing=true'], 'env': [{'name': 'OTEL_TRACES_EXPORTER', 'value': 'otlp'}, {'name': 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', 'value': 'http://data-science-collector-collector.opendatahub.svc:4317'}, {'name': 'OTEL_EXPORTER_OTLP_ENDPOINT', 'value': 'http://data-science-collector-collector.opendatahub.svc:4317'}, {'name': 'OTEL_EXPORTER_OTLP_INSECURE', 'value': 'true'}, {'name': 'OTEL_TRACES_SAMPLER', 'value': 'parentbased_traceidratio'}, {'name': 'OTEL_SERVICE_NAME', 'value': 'payload-processing'}, {'name': 'OTEL_TRACES_SAMPLER_ARG', 'value': '1.0'}, {'name': 'GATEWAY_NAMESPACE', 'value': 'openshift-ingress'}, {'name': 'GATEWAY_NAME', 'value': 'e2e-ipp-b-c91909'}, {'name': 'TENANT_NAMESPACE', 'value': 'ai-tenant-e2e-ipp-b-c91909'}, {'name': 'DISABLE_EXTERNAL_MODEL_CONTROLLER', 'value': 'true'}], 'image': 'quay.io/opendatahub/odh-ai-gateway-payload-processing:odh-stable', 'imagePullPolicy': 'IfNotPresent', 'livenessProbe': {'failureThreshold': 3, 'initialDelaySeconds': 15, 'periodSeconds': 20, 'successThreshold': 1, 'tcpSocket': {'port': 'grpc'}, 'timeoutSeconds': 1}, 'name': 'payload-processing', 'ports': [{'containerPort': 9004, 'name': 'grpc', 'protocol': 'TCP'}, {'containerPort': 9005, 'name': 'metrics', 'protocol': 'TCP'}], 'readinessProbe': {'failureThreshold': 3, 'initialDelaySeconds': 5, 'periodSeconds': 10, 'successThreshold': 1, 'tcpSocket': {'port': 'grpc'}, 'timeoutSeconds': 1}, 'resources': {'limits': {'cpu': '1', 'memory': '512Mi'}, 'requests': {'cpu': '100m', 'memory': '128Mi'}}, 'securityContext': {'allowPrivilegeEscalation': False, 'capabilities': {'drop': ['ALL']}, 'readOnlyRootFilesystem': True, 'seccompProfile': {'type': 'RuntimeDefault'}}, 'terminationMessagePath': '/dev/termination-log', 'terminationMessagePolicy': 'File', 'volumeMounts': [{'mountPath': '/config', 'name': 'plugins-config-volume'}]}], 'dnsPolicy': 'ClusterFirst', 'restartPolicy': 'Always', 'schedulerName': 'default-scheduler', 'securityContext': {'runAsNonRoot': True}, 'serviceAccount': 'payload-processing-e2e-ipp-b-c91909', 'serviceAccountName': 'payload-processing-e2e-ipp-b-c91909', 'terminationGracePeriodSeconds': 30, 'volumes': [{'configMap': {'defaultMode': 420, 'name': 'payload-processing-plugins-e2e-ipp-b-c91909'}, 'name': 'plugins-config-volume'}]}}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T13:34:34Z', 'lastUpdateTime': '2026-10-30T13:34:34Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'Available'}, {'lastTransitionTime': '2026-10-30T13:34:34Z', 'lastUpdateTime': '2026-10-30T13:34:35Z', 'message': 'ReplicaSet "payload-processing-e2e-ipp-b-c91909-574f7f7cf8" is progressing.', 'reason': 'ReplicaSetUpdated', 'status': 'True', 'type': 'Progressing'}], 'observedGeneration': 1, 'replicas': 1, 'terminatingReplicas': 0, 'unavailableReplicas': 1, 'updatedReplicas': 1}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError@pytest.fixture(scope="module") def ipp_tenant_cases(): require_tenant_namespace_discovery() require_aitenant_crd() case_a = new_named_tenant_case("e2e-ipp-a") case_b = new_named_tenant_case("e2e-ipp-b") try: for case in (case_a, case_b): bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP) > wait_for_per_tenant_ipp_ready(case) test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:125: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1336: in wait_for_per_tenant_ipp_ready wait_for_deployment_available(names["processing_deployment"], GATEWAY_NAMESPACE, timeout=timeout) test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:428: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'payload-processing-e2e-ipp-b-c91909' namespace = 'openshift-ingress' predicate = <function wait_for_deployment_available.<locals>._predicate at 0x7f48be72ea20> timeout = 240, interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/payload-processing-e2e-ipp-b-c91909 in openshift-ingress did not satisfy condition. Last object: {'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': {'annotations': {'deployment.kubernetes.io/revision': '1'}, 'creationTimestamp': '2026-10-30T13:34:34Z', 'generation': 1, 'labels': {'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service', 'app.opendatahub.io/modelsasservice': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-ipp-b-c91909', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-ipp-b-c91909'}, 'name': 'payload-processing-e2e-ipp-b-c91909', 'namespace': 'openshift-ingress', 'ownerReferences': [{'apiVersion': 'maas.opendatahub.io/v1alpha1', 'blockOwnerDeletion': True, 'controller': True, 'kind': 'Config', 'name': 'default', 'uid': 'b7a60aef-fc0c-4831-96cb-da865b6e4ee8'}], 'resourceVersion': '116914', 'uid': 'c8aea13d-acdc-406d-998a-5c513168ac24'}, 'spec': {'progressDeadlineSeconds': 600, 'replicas': 1, 'revisionHistoryLimit': 10, 'selector': {'matchLabels': {'app': 'payload-processing', 'maas.opendatahub.io/tenant-instance': 'payload-processing-e2e-ipp-b-c91909'}}, 'strategy': {'rollingUpdate': {'maxSurge': '25%', 'maxUnavailable': '25%'}, 'type': 'RollingUpdate'}, 'template': {'metadata': {'labels': {'app': 'payload-processing', 'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service', 'maas.opendatahub.io/tenant-instance': 'payload-processing-e2e-ipp-b-c91909'}}, 'spec': {'containers': [{'args': ['--config-file', '/config/custom-ipp-config.yaml', '--v=3', '--tracing=true'], 'env': [{'name': 'OTEL_TRACES_EXPORTER', 'value': 'otlp'}, {'name': 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT', 'value': 'http://data-science-collector-collector.opendatahub.svc:4317'}, {'name': 'OTEL_EXPORTER_OTLP_ENDPOINT', 'value': 'http://data-science-collector-collector.opendatahub.svc:4317'}, {'name': 'OTEL_EXPORTER_OTLP_INSECURE', 'value': 'true'}, {'name': 'OTEL_TRACES_SAMPLER', 'value': 'parentbased_traceidratio'}, {'name': 'OTEL_SERVICE_NAME', 'value': 'payload-processing'}, {'name': 'OTEL_TRACES_SAMPLER_ARG', 'value': '1.0'}, {'name': 'GATEWAY_NAMESPACE', 'value': 'openshift-ingress'}, {'name': 'GATEWAY_NAME', 'value': 'e2e-ipp-b-c91909'}, {'name': 'TENANT_NAMESPACE', 'value': 'ai-tenant-e2e-ipp-b-c91909'}, {'name': 'DISABLE_EXTERNAL_MODEL_CONTROLLER', 'value': 'true'}], 'image': 'quay.io/opendatahub/odh-ai-gateway-payload-processing:odh-stable', 'imagePullPolicy': 'IfNotPresent', 'livenessProbe': {'failureThreshold': 3, 'initialDelaySeconds': 15, 'periodSeconds': 20, 'successThreshold': 1, 'tcpSocket': {'port': 'grpc'}, 'timeoutSeconds': 1}, 'name': 'payload-processing', 'ports': [{'containerPort': 9004, 'name': 'grpc', 'protocol': 'TCP'}, {'containerPort': 9005, 'name': 'metrics', 'protocol': 'TCP'}], 'readinessProbe': {'failureThreshold': 3, 'initialDelaySeconds': 5, 'periodSeconds': 10, 'successThreshold': 1, 'tcpSocket': {'port': 'grpc'}, 'timeoutSeconds': 1}, 'resources': {'limits': {'cpu': '1', 'memory': '512Mi'}, 'requests': {'cpu': '100m', 'memory': '128Mi'}}, 'securityContext': {'allowPrivilegeEscalation': False, 'capabilities': {'drop': ['ALL']}, 'readOnlyRootFilesystem': True, 'seccompProfile': {'type': 'RuntimeDefault'}}, 'terminationMessagePath': '/dev/termination-log', 'terminationMessagePolicy': 'File', 'volumeMounts': [{'mountPath': '/config', 'name': 'plugins-config-volume'}]}], 'dnsPolicy': 'ClusterFirst', 'restartPolicy': 'Always', 'schedulerName': 'default-scheduler', 'securityContext': {'runAsNonRoot': True}, 'serviceAccount': 'payload-processing-e2e-ipp-b-c91909', 'serviceAccountName': 'payload-processing-e2e-ipp-b-c91909', 'terminationGracePeriodSeconds': 30, 'volumes': [{'configMap': {'defaultMode': 420, 'name': 'payload-processing-plugins-e2e-ipp-b-c91909'}, 'name': 'plugins-config-volume'}]}}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T13:34:34Z', 'lastUpdateTime': '2026-10-30T13:34:34Z', 'message': 'Deployment does not have minimum availability.', 'reason': 'MinimumReplicasUnavailable', 'status': 'False', 'type': 'Available'}, {'lastTransitionTime': '2026-10-30T13:34:34Z', 'lastUpdateTime': '2026-10-30T13:34:35Z', 'message': 'ReplicaSet "payload-processing-e2e-ipp-b-c91909-574f7f7cf8" is progressing.', 'reason': 'ReplicaSetUpdated', 'status': 'True', 'type': 'Progressing'}], 'observedGeneration': 1, 'replicas': 1, 'terminatingReplicas': 0, 'unavailableReplicas': 1, 'updatedReplicas': 1}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError@pytest.fixture(scope="module") def ipp_tenant_cases(): require_tenant_namespace_discovery() require_aitenant_crd() case_a = new_named_tenant_case("e2e-ipp-a") case_b = new_named_tenant_case("e2e-ipp-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP) test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ipp-b-72208f', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f8b01072160>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-ipp-b-72208f in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-b-72208f","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-b-72208f"}}}\n'}, 'creationTimestamp': '2026-10-30T13:33:40Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-b-72208f', 'namespace': 'ai-tenants', 'resourceVersion': '115074', 'uid': '5de9b3a9-05e8-4e1d-afd1-71df2231c38f'}, 'spec': {'gateway': {'name': 'e2e-ipp-b-72208f'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T13:33:42Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-b-72208f', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-b-72208f'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError@pytest.fixture(scope="module") def ipp_tenant_cases(): require_tenant_namespace_discovery() require_aitenant_crd() case_a = new_named_tenant_case("e2e-ipp-a") case_b = new_named_tenant_case("e2e-ipp-b") try: for case in (case_a, case_b): > bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP) test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'aitenant', name = 'e2e-ipp-b-72208f', namespace = 'ai-tenants' predicate = <function aitenant_ready at 0x7f8b01072160>, timeout = 180 interval = 5 def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: aitenant/e2e-ipp-b-72208f in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-b-72208f","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-b-72208f"}}}\n'}, 'creationTimestamp': '2026-10-30T13:33:40Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-b-72208f', 'namespace': 'ai-tenants', 'resourceVersion': '115074', 'uid': '5de9b3a9-05e8-4e1d-afd1-71df2231c38f'}, 'spec': {'gateway': {'name': 'e2e-ipp-b-72208f'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T13:33:42Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-b-72208f', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-b-72208f'}} test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError