<?xml version="1.0" encoding="utf-8"?><testsuites name="pytest tests"><testsuite name="pytest" errors="3" failures="2" skipped="17" tests="203" time="2215.833" timestamp="2026-10-30T00:55:47.591308+00:00" hostname="ai-gateway-controller-group0f2b0a0e955790295f2220d5d451883d-pod"><testcase classname="tests.test_api_keys.TestAPIKeyCRUD" name="test_create_api_key" time="100.539" /><testcase classname="tests.test_api_keys.TestAPIKeyCRUD" name="test_list_api_keys" time="101.305" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_admin_can_revoke_any_user" time="0.899" /><testcase classname="tests.test_api_keys.TestAPIKeyExpiration" name="test_expiration_policy_boundaries[within-limit]" time="0.089" /><testcase classname="tests.test_api_keys.TestAPIKeyExpiration" name="test_expiration_policy_boundaries[exceeds-limit]" time="0.095" /><testcase classname="tests.test_api_keys.TestAPIKeyExpiration" name="test_expiration_policy_boundaries[without-expiration]" time="0.085" /><testcase classname="tests.test_api_keys.TestAPIKeyExpiration" name="test_expiration_policy_boundaries[short-expiration]" time="0.093" /><testcase classname="tests.test_api_keys.TestAPIKeyModelInference" name="test_api_key_model_access_success" time="0.190" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_by_subscription" time="25.524" /><testcase classname="tests.test_api_keys.TestAPIKeyModelInference" name="test_invalid_api_key_rejected" time="0.052" /><testcase classname="tests.test_api_keys.TestAPIKeyModelInference" name="test_no_auth_header_rejected" time="0.037" /><testcase classname="tests.test_api_keys.TestAPIKeyModelInference" name="test_revoked_api_key_rejected" time="2.308" /><testcase classname="tests.test_api_keys.TestAPIKeyModelInference" name="test_api_key_chat_completions" time="0.086" /><testcase classname="tests.test_api_keys.TestAPIKeyRevocationE2E" name="test_double_revoke_returns_404" time="0.264" /><testcase classname="tests.test_api_keys.TestAPIKeyRevocationE2E" name="test_revoke_nonexistent_key_returns_404" time="0.085" /><testcase classname="tests.test_api_keys.TestAPIKeyRevocationE2E" name="test_revoke_then_create_new_key_works" time="0.432" /><testcase classname="tests.test_api_keys.TestAPIKeyRevocationE2E" name="test_individual_revoke_multiple_keys" time="0.597" /><testcase classname="tests.test_api_keys.TestAPIKeyRevocationE2E" name="test_revoke_keys_rejected_at_gateway" time="0.851" /><testcase classname="tests.test_api_keys.TestEphemeralKeyCleanup" name="test_cronjob_exists_and_configured" time="0.155" /><testcase classname="tests.test_api_keys.TestEphemeralKeyCleanup" name="test_cleanup_networkpolicy_exists" time="0.141" /><testcase classname="tests.test_api_keys.TestEphemeralKeyCleanup" name="test_create_ephemeral_key" time="0.483" /><testcase classname="tests.test_api_keys.TestEphemeralKeyCleanup" name="test_trigger_cleanup_preserves_active_keys" time="0.247"><skipped type="pytest.skip" message="Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath &quot;{.items[0].metadata.name}&quot;: Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template:&#10;&#09;template was:&#10;&#09;&#09;{.items[0].metadata.name}&#10;&#09;object given to jsonpath engine was:&#10;&#09;&#09;map[string]interface {}{&quot;apiVersion&quot;:&quot;v1&quot;, &quot;items&quot;:[]interface {}{}, &quot;kind&quot;:&quot;List&quot;, &quot;metadata&quot;:map[string]interface {}{&quot;resourceVersion&quot;:&quot;&quot;}}">/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template:
	template was:
		{.items[0].metadata.name}
	object given to jsonpath engine was:
		map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}</skipped></testcase><testcase classname="tests.test_api_keys.TestAPIKeySubscriptionPhases" name="test_create_key_for_active_subscription" time="12.221" /><testcase classname="tests.test_api_keys.TestAPIKeyAuthorization" name="test_non_admin_cannot_access_other_users_keys" time="111.256" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_dry_run_by_subscription" time="12.515" /><testcase classname="tests.test_api_keys.TestAPIKeySubscriptionPhases" name="test_create_key_for_degraded_subscription" time="14.345" /><testcase classname="tests.test_api_keys.TestAPIKeySubscriptionPhases" name="test_create_key_for_failed_subscription" time="12.391" /><testcase classname="tests.test_api_keys.TestAPIKeyAuthorization" name="test_admin_manage_other_users_keys" time="125.653" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_other_user_forbidden" time="125.275" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_own_keys" time="126.422" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_missing_scope_returns_400" time="0.151" /><testcase classname="tests.test_api_keys.TestAPIKeyLabels" name="test_create_api_key_with_labels" time="0.253" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_dry_run" time="0.785" /><testcase classname="tests.test_api_keys.TestAPIKeyLabels" name="test_search_api_keys_by_labels" time="0.740" /><testcase classname="tests.test_api_keys.TestAPIKeyLabels" name="test_backward_compatibility_no_labels" time="0.259" /><testcase classname="tests.test_api_keys.TestAPIKeyLabels" name="test_labels_validation_errors" time="0.661" /><testcase classname="tests.test_api_keys.TestAPIKeyCRUD" name="test_revoke_api_key" time="127.042" /><testcase classname="tests.test_namespace_scoping.TestMaaSAPIWatchNamespace" name="test_subscription_in_another_namespace_not_visible_to_api" time="0.000"><skipped type="pytest.skip" message="test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true">/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true</skipped></testcase><testcase classname="tests.test_namespace_scoping.TestMaaSControllerWatchNamespace" name="test_authpolicy_and_subscription_in_another_namespace" time="0.000"><skipped type="pytest.skip" message="test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true">/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true</skipped></testcase><testcase classname="tests.test_namespace_scoping.TestModelRef" name="test_auth_policy_model_ref" time="0.003"><skipped type="pytest.skip" message="test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true">/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true</skipped></testcase><testcase classname="tests.test_namespace_scoping.TestModelRef" name="test_subscription_model_ref" time="0.001"><skipped type="pytest.skip" message="test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true">/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true</skipped></testcase><testcase classname="tests.test_negative_security.TestAPIKeyManagementIsolation" name="test_api_key_cannot_mint_another_api_key" time="0.612" /><testcase classname="tests.test_api_keys.TestAPIKeyExpiration" name="test_expiration_policy_boundaries[at-limit]" time="0.514" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_combined_user_and_subscription" time="16.309" /><testcase classname="tests.test_api_keys.TestAPIKeyBulkOperations" name="test_bulk_revoke_by_subscription_forbidden_for_non_admin" time="0.144" /><testcase classname="tests.test_namespace_scoping.TestMaaSControllerWatchNamespace" name="test_authpolicy_and_subscription_in_maas_subscription_namespace" time="0.000"><skipped type="pytest.skip" message="test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true">/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true</skipped></testcase><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_forged_identity_headers_rejected_on_key_mint[identity-envelope]" time="0.965" /><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_forged_identity_headers_rejected_on_key_mint[keyname-only]" time="0.882" /><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_forged_identity_headers_rejected_on_key_mint[username-only]" time="0.833" /><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_forged_identity_headers_rejected_on_key_mint[group-only]" time="0.819" /><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_injected_identity_headers_rejected_on_inference" time="0.358" /><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_forged_identity_headers_rejected_on_key_mint[admin-scope]" time="0.775" /><testcase classname="tests.test_negative_security.TestHeaderSpoofing" name="test_duplicate_subscription_headers_ignored" time="8.507" /><testcase classname="tests.test_negative_security.TestExpiredKeyRejection" name="test_expired_key_rejected_at_gateway" time="5.190" /><testcase classname="tests.test_negative_security.TestCrossModelAccess" name="test_key_cannot_access_model_outside_subscription" time="0.354" /><testcase classname="tests.test_negative_security.TestInternalEndpointIsolation" name="test_internal_endpoint_not_routable[subscriptions-select]" time="0.195" /><testcase classname="tests.test_negative_security.TestMissingModelRef" name="test_subscription_with_nonexistent_model_ref" time="13.560" /><testcase classname="tests.test_negative_security.TestInternalEndpointIsolation" name="test_internal_endpoint_not_routable[api-keys-cleanup]" time="0.200" /><testcase classname="tests.test_negative_security.TestInternalEndpointIsolation" name="test_health_endpoint_accessible" time="0.034" /><testcase classname="tests.test_negative_security.TestInternalEndpointIsolation" name="test_v1_models_via_maas_api_prefix" time="0.265" /><testcase classname="tests.test_subscription.TestAuthEnforcement" name="test_authorized_user_gets_200" time="0.214" /><testcase classname="tests.test_subscription.TestAuthEnforcement" name="test_no_auth_gets_401" time="0.041" /><testcase classname="tests.test_subscription.TestAuthEnforcement" name="test_invalid_token_gets_403" time="0.127" /><testcase classname="tests.test_subscription.TestAuthEnforcement" name="test_wrong_group_gets_403" time="0.051" /><testcase classname="tests.test_subscription.TestAPIKeySubscriptionBinding" name="test_create_api_key_uses_highest_priority_subscription" time="2.648" /><testcase classname="tests.test_subscription.TestAPIKeySubscriptionBinding" name="test_create_api_key_with_explicit_simulator_subscription" time="0.210" /><testcase classname="tests.test_subscription.TestAPIKeySubscriptionBinding" name="test_create_api_key_nonexistent_subscription_errors" time="10.401" /><testcase classname="tests.test_negative_security.TestHeaderAbuse" name="test_special_characters_in_subscription_header" time="0.649" /><testcase classname="tests.test_subscription.TestOrderingEdgeCases" name="test_subscription_before_auth_policy" time="15.551" /><testcase classname="tests.test_api_keys.TestAPIKeySubscriptionFilter" name="test_search_filters_by_subscription" time="31.105" /><testcase classname="tests.test_api_keys.TestAPIKeySubscriptionFilter" name="test_search_without_subscription_returns_all" time="1.026" /><testcase classname="tests.test_negative_security.TestMissingModelRef" name="test_authpolicy_with_nonexistent_model_ref" time="3.122" /><testcase classname="tests.test_subscription.TestE2ESubscriptionFlow" name="test_e2e_with_both_access_and_subscription_gets_200" time="197.025"><failure message="TimeoutError: TokenRateLimitPolicy maas-trlp-premium-w5-dca38d was not created and enforced in e2e-models-e2e-worker-w5-dca38d within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T00:58:21Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True">self = &lt;test_subscription.TestE2ESubscriptionFlow object at 0x7f75d1564610&gt;

    def test_e2e_with_both_access_and_subscription_gets_200(self):
        """
        API key with both access and subscription should get 200 OK.
    
        Uses an existing model reference that aligns with gateway path-based model identity.
        """
        ns = _ns()
        auth_policy_name = "e2e-test-auth-success"
        subscription_name = "e2e-test-subscription-success"
        sa_name = "e2e-sa-success"
    
        try:
            # Create service account and get OC token for maas-api
            oc_token = _create_sa_token(sa_name, namespace=ns)
            sa_user = _sa_to_user(sa_name, namespace=ns)
    
            # Use pre-existing premium model to keep model identity consistent with gateway path.
            _create_test_auth_policy(auth_policy_name, PREMIUM_MODEL_REF, users=[sa_user])
            _create_test_subscription(subscription_name, PREMIUM_MODEL_REF, users=[sa_user])
            _wait_for_maas_auth_policy_phase(auth_policy_name, namespace=ns, timeout=120, require_auth_policies=False)
&gt;           _wait_for_subscription_inference_ready(
                subscription_name,
                PREMIUM_MODEL_REF,
                namespace=ns,
                model_namespace=MODEL_NAMESPACE,
                timeout=180,
            )

test/maas-e2e/test/e2e/tests/test_subscription.py:1887: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready
    _wait_for_token_rate_limit_policy(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

model_ref = 'premium-w5-dca38d'
model_namespace = 'e2e-models-e2e-worker-w5-dca38d', timeout = 180
subscription_name = 'e2e-test-subscription-success'
subscription_namespace = 'ai-tenant-e2e-worker-w5-dca38d'

    def _wait_for_token_rate_limit_policy(
        model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None
    ):
        """Wait for the current TokenRateLimitPolicy generation to be enforced.
    
        If a subscription is supplied, wait until the policy includes it. This
        prevents accepting the old policy before the controller updates its spec.
    
        Args:
            model_ref: Name of the model (e.g., "e2e-distinct-simulated")
            model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE)
            timeout: Maximum wait time in seconds (default: 60)
            subscription_name: Subscription that the policy must include, if supplied
            subscription_namespace: Subscription namespace (defaults to _ns())
    
        Raises:
            TimeoutError: If TRLP isn't created and enforced within timeout
        """
        trlp_name = f"maas-trlp-{model_ref}"
        expected_subscription = (
            f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None
        )
        last_snapshot = "not found"
        deadline = time.time() + timeout
        log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...")
    
        while time.time() &lt; deadline:
            result = subprocess.run(
                ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode == 0:
                try:
                    trlp = json.loads(result.stdout)
                    metadata = trlp.get("metadata", {})
                    status = trlp.get("status", {})
                    generation = metadata.get("generation")
                    observed = status.get("observedGeneration")
                    conditions = status.get("conditions", [])
                    enforced = next((c for c in conditions if c.get("type") == "Enforced"), None)
                    subscriptions = {
                        name.strip()
                        for name in metadata.get("annotations", {}).get(
                            "maas.opendatahub.io/subscriptions", ""
                        ).split(",")
                    }
                    includes_subscription = expected_subscription is None or expected_subscription in subscriptions
                    last_snapshot = (
                        f"generation={generation}, observedGeneration={observed}, "
                        f"Enforced={enforced}, includes_subscription={includes_subscription}"
                    )
                    if (
                        enforced and enforced.get("status") == "True"
                        and generation is not None and observed == generation
                        and includes_subscription
                    ):
                        log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced")
                        return
                    log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}")
                except (json.JSONDecodeError, KeyError) as e:
                    log.debug(f"Failed to parse TRLP status: {e}")
            elif _is_not_found_error(result.stderr):
                log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...")
            elif _is_transient_kubectl_error(result.stderr):
                log.debug(
                    f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}"
                )
            else:
                raise RuntimeError(
                    f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': "
                    f"{result.stderr.strip()}"
                )
            time.sleep(3)
    
&gt;       raise TimeoutError(
            f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}"
        )
E       TimeoutError: TokenRateLimitPolicy maas-trlp-premium-w5-dca38d was not created and enforced in e2e-models-e2e-worker-w5-dca38d within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T00:58:21Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True

test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError</failure></testcase><testcase classname="tests.test_subscription.TestE2ESubscriptionFlow" name="test_e2e_with_subscription_but_no_access_gets_403" time="15.690" /><testcase classname="tests.test_negative_security.TestInternalEndpointIsolation" name="test_internal_endpoint_not_routable[api-keys-validate]" time="0.196" /><testcase classname="tests.test_namespace_scoping.TestMaaSAPIWatchNamespace" name="test_subscription_in_subscription_namespace_visible_to_api" time="0.000"><skipped type="pytest.skip" message="test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true">/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true</skipped></testcase><testcase classname="tests.test_subscription.TestE2ESubscriptionFlow" name="test_e2e_group_based_access_gets_200" time="194.440"><failure message="TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w3-82b647 was not created and enforced in e2e-models-e2e-worker-w3-82b647 within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T00:58:22Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True">self = &lt;test_subscription.TestE2ESubscriptionFlow object at 0x7f592c2aa010&gt;

    def test_e2e_group_based_access_gets_200(self):
        """
        E2E test: Group-based auth and subscription (success case).
    
        Validates that users can access models via group membership in both
        MaaSAuthPolicy and MaaSSubscription, not just explicit user lists.
        """
        ns = _ns()
        auth_policy_name = "e2e-test-group-auth"
        subscription_name = "e2e-test-group-subscription"
        sa_name = "e2e-sa-group"
    
        # Use namespace-specific group that SA will be in
        test_group = f"system:serviceaccounts:{ns}"
    
        try:
            # Create service account and get OC token for maas-api
            oc_token = _create_sa_token(sa_name, namespace=ns)
    
            # Create auth policy using GROUP (not user)
            _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group])
    
            # Create subscription using GROUP (not user)
            _create_test_subscription(subscription_name, MODEL_REF, groups=[test_group])
    
            _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False)
&gt;           _wait_for_subscription_inference_ready(
                subscription_name,
                MODEL_REF,
                model_namespace=MODEL_NAMESPACE,
                timeout=180,
            )

test/maas-e2e/test/e2e/tests/test_subscription.py:2216: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready
    _wait_for_token_rate_limit_policy(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

model_ref = 'simulator-w3-82b647'
model_namespace = 'e2e-models-e2e-worker-w3-82b647', timeout = 180
subscription_name = 'e2e-test-group-subscription'
subscription_namespace = 'ai-tenant-e2e-worker-w3-82b647'

    def _wait_for_token_rate_limit_policy(
        model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None
    ):
        """Wait for the current TokenRateLimitPolicy generation to be enforced.
    
        If a subscription is supplied, wait until the policy includes it. This
        prevents accepting the old policy before the controller updates its spec.
    
        Args:
            model_ref: Name of the model (e.g., "e2e-distinct-simulated")
            model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE)
            timeout: Maximum wait time in seconds (default: 60)
            subscription_name: Subscription that the policy must include, if supplied
            subscription_namespace: Subscription namespace (defaults to _ns())
    
        Raises:
            TimeoutError: If TRLP isn't created and enforced within timeout
        """
        trlp_name = f"maas-trlp-{model_ref}"
        expected_subscription = (
            f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None
        )
        last_snapshot = "not found"
        deadline = time.time() + timeout
        log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...")
    
        while time.time() &lt; deadline:
            result = subprocess.run(
                ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"],
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode == 0:
                try:
                    trlp = json.loads(result.stdout)
                    metadata = trlp.get("metadata", {})
                    status = trlp.get("status", {})
                    generation = metadata.get("generation")
                    observed = status.get("observedGeneration")
                    conditions = status.get("conditions", [])
                    enforced = next((c for c in conditions if c.get("type") == "Enforced"), None)
                    subscriptions = {
                        name.strip()
                        for name in metadata.get("annotations", {}).get(
                            "maas.opendatahub.io/subscriptions", ""
                        ).split(",")
                    }
                    includes_subscription = expected_subscription is None or expected_subscription in subscriptions
                    last_snapshot = (
                        f"generation={generation}, observedGeneration={observed}, "
                        f"Enforced={enforced}, includes_subscription={includes_subscription}"
                    )
                    if (
                        enforced and enforced.get("status") == "True"
                        and generation is not None and observed == generation
                        and includes_subscription
                    ):
                        log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced")
                        return
                    log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}")
                except (json.JSONDecodeError, KeyError) as e:
                    log.debug(f"Failed to parse TRLP status: {e}")
            elif _is_not_found_error(result.stderr):
                log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...")
            elif _is_transient_kubectl_error(result.stderr):
                log.debug(
                    f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}"
                )
            else:
                raise RuntimeError(
                    f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': "
                    f"{result.stderr.strip()}"
                )
            time.sleep(3)
    
&gt;       raise TimeoutError(
            f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}"
        )
E       TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w3-82b647 was not created and enforced in e2e-models-e2e-worker-w3-82b647 within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T00:58:22Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True

test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError</failure></testcase><testcase classname="tests.test_subscription.TestSubscriptionEnforcement" name="test_subscribed_user_gets_200" time="0.266" /><testcase classname="tests.test_subscription.TestManagedAnnotation" name="test_authpolicy_managed_false_prevents_update" time="0.304"><skipped type="pytest.skip" message="gateway-only mode: per-model AuthPolicy is not created">/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1580: gateway-only mode: per-model AuthPolicy is not created</skipped></testcase><testcase classname="tests.test_subscription.TestE2ESubscriptionFlow" name="test_e2e_mint_api_key_denied_for_inaccessible_subscription" time="23.555" /><testcase classname="tests.test_subscription.TestManagedAnnotation" name="test_trlp_managed_false_prevents_update" time="301.136" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_subscription_active_status_with_valid_model" time="76.177" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_subscription_failed_status_with_missing_model" time="11.265" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_authpolicy_active_status_with_valid_model" time="96.314" /><testcase classname="tests.test_subscription.TestDegradedSubscriptionFiltering" name="test_models_endpoint_with_degraded_subscription_kube_token" time="14.282" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_authpolicy_failed_status_with_missing_model" time="3.301" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_subscription_degraded_status_with_partial_models" time="108.624" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptions" name="test_unauthenticated_returns_401[all-subscriptions]" time="4.863" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_subscription_status_transitions_on_model_deletion" time="99.240" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptions" name="test_unauthenticated_returns_401[subscriptions-for-model]" time="0.089" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptions" name="test_model_ref_display_name_and_description_enriched" time="12.421" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptions" name="test_returns_accessible_subscriptions" time="5.517" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptions" name="test_subscription_includes_model_refs" time="11.628" /><testcase classname="tests.test_subscription.TestStatusReporting" name="test_authpolicy_degraded_status_with_partial_models" time="3.523" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptionsForModel" name="test_returns_subscriptions_for_model" time="34.187" /><testcase classname="tests.test_subscription_list_endpoints.TestListSubscriptionsForModel" name="test_unknown_model_returns_empty" time="1.032" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_empty_subscription_header_value" time="6.794" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_multiple_distinct_models_in_subscription" time="24.905" /><testcase classname="tests.test_subscription_list_endpoints.TestSubscriptionModelAccessFiltering" name="test_filters_unauthorized_models" time="24.667" /><testcase classname="tests.test_subscription.TestDegradedSubscriptionFiltering" name="test_models_endpoint_with_degraded_subscription_api_key" time="15.137" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_deduplication_same_model_multiple_refs" time="21.846" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_user_token_returns_all_models" time="25.526" /><testcase classname="tests.test_subscription_list_endpoints.TestSubscriptionModelAccessFiltering" name="test_omits_subscription_with_no_authorized_models" time="12.277" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_user_token_with_subscription_header_filters" time="22.107" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_response_schema_and_metadata" time="0.666" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_access_denied_to_subscription_403" time="23.438" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_api_key_with_inaccessible_subscription_403" time="17.056" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_api_key_scoped_to_subscription" time="16.715" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_api_key_with_deleted_subscription_403" time="42.361" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_invalid_subscription_header_403" time="12.220" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_multiple_api_keys_different_subscriptions" time="29.520" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_api_key_ignores_subscription_header" time="25.024" /><testcase classname="tests.test_subscription.TestE2ESubscriptionFlow" name="test_e2e_multiple_subscriptions_separate_keys_gets_200" time="153.049" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_service_account_token_multiple_subs_with_header" time="36.164" /><testcase classname="tests.test_subscription.TestE2ESubscriptionFlow" name="test_e2e_group_based_subscription_but_no_auth_gets_403" time="12.276" /><testcase classname="tests.test_smoke" name="test_chat_completions_gateway_alive" time="0.317" /><testcase classname="tests.test_smoke" name="test_chat_completions_streaming_sse" time="0.107" /><testcase classname="tests.test_smoke" name="test_legacy_completions_optionally" time="0.184" /><testcase classname="tests.test_tenant.TestTenantLifecycle" name="test_tenant_ready_and_phase_healthy" time="0.456" /><testcase classname="tests.test_tenant.TestTenantContract" name="test_status_has_phase_and_conditions" time="0.142" /><testcase classname="tests.test_tenant.TestTenantContract" name="test_spec_is_well_formed" time="0.142" /><testcase classname="tests.test_tenant.TestTenantContract" name="test_conditions_use_kubernetes_metav1_shape" time="0.139" /><testcase classname="tests.test_tenant.TestTenantNoFalseOwnership" name="test_maas_user_crs_not_owned_by_tenant" time="0.470" /><testcase classname="tests.test_config_tenant.TestConfigAnchorPresence" name="test_cluster_config_default_exists" time="0.545" /><testcase classname="tests.test_config_tenant.TestConfigAnchorPresence" name="test_cluster_config_not_terminating" time="0.190" /><testcase classname="tests.test_config_tenant.TestConfigTenantOwnership" name="test_default_aitenant_lists_config_owner_reference" time="0.144" /><testcase classname="tests.test_config_tenant.TestConfigTenantOwnership" name="test_tenant_config_lists_config_owner_reference" time="0.149" /><testcase classname="tests.test_config_tenant.TestConfigTenantOwnership" name="test_maas_controller_deployment_does_not_list_config_owner_reference" time="0.152" /><testcase classname="tests.test_tenant_discovery" name="test_tenant_discovery_requires_auth" time="2.441" /><testcase classname="tests.test_tenant_discovery" name="test_tenant_discovery_with_invalid_token" time="2.018" /><testcase classname="tests.test_tenant_discovery" name="test_tenant_discovery_authenticated" time="0.001"><skipped type="pytest.skip" message="Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.</skipped></testcase><testcase classname="tests.test_tenant_discovery" name="test_tenant_discovery_gateway_matches_deployment" time="0.001"><skipped type="pytest.skip" message="Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.</skipped></testcase><testcase classname="tests.test_tenant_discovery" name="test_tenant_discovery_not_exposed_through_gateway" time="0.036" /><testcase classname="tests.test_aitenant_lifecycle.TestAITenantLifecycle" name="test_default_aitenant_bootstraps_maas_tenant_config_without_gateway_mutation" time="2.180" /><testcase classname="tests.test_smoke" name="test_healthz_or_404" time="0.037" /><testcase classname="tests.test_aitenant_lifecycle.TestAITenantLifecycle" name="test_aitenant_create_bootstrap_resources" time="62.052" /><testcase classname="tests.test_aitenant_lifecycle.TestAITenantLifecycle" name="test_aitenant_rejected_outside_ai_tenants_namespace" time="7.161" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_unauthenticated_request_401" time="0.052" /><testcase classname="tests.test_models_endpoint.TestModelsEndpoint" name="test_service_account_token_multiple_subs_no_header" time="24.385" /><testcase classname="tests.test_aitenant_lifecycle.TestAITenantLifecycle" name="test_aitenant_derives_non_default_tenant_namespace" time="63.039" /><testcase classname="tests.test_aitenant_lifecycle.TestAITenantLifecycle" name="test_aitenant_delete_cleans_maas_resources_and_preserves_user_objects" time="64.916" /><testcase classname="tests.test_smoke" name="test_models_catalog" time="0.598" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_dynamic_discovery_after_label_added" time="107.518" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_label_removal_stops_reconciliation" time="117.730" /><testcase classname="tests.test_aitenant_lifecycle.TestAITenantLifecycle" name="test_aitenant_migrates_and_removes_legacy_tenant" time="65.113" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_labeled_tenant_namespace_is_discovered" time="102.751" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_unlabeled_namespace_ignored" time="106.928" /><testcase classname="tests.test_subscription.TestDegradedSubscriptionFiltering" name="test_degraded_healthy_model_allows_inference" time="14.312" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_per_tenant_oidc_configuration" time="0.001"><skipped type="pytest.skip" message="OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy</skipped></testcase><testcase classname="tests.test_tenant_namespace_discovery.TestLegacyDefaultNamespaceStillWorks" name="test_models_as_a_service_namespace_reconciles" time="6.451" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_tenant_admin_rbac_is_namespace_scoped" time="193.783" /><testcase classname="tests.test_smoke" name="test_tokens_endpoint_replaced_by_api_keys" time="0.142" /><testcase classname="tests.test_gateway_scoped_authpolicy.TestGatewayAuthPolicyStructure" name="test_gateway_auth_policy_contract" time="0.480" /><testcase classname="tests.test_gateway_scoped_authpolicy.TestGatewayAuthPolicyLifecycle" name="test_only_one_gateway_authpolicy_named_maas_gateway_auth" time="0.328" /><testcase classname="tests.test_gateway_scoped_authpolicy.TestGatewayAuthPolicyManagementEndpointAccess" name="test_management_rules_have_model_identity_guards" time="0.157" /><testcase classname="tests.test_gateway_scoped_authpolicy.TestGatewayAuthPolicyManagementEndpointAccess" name="test_gateway_default_auth_scoped_if_present" time="0.001"><skipped type="pytest.skip" message="legacy gateway-default-auth is scoped to the shared default gateway">/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gateway</skipped></testcase><testcase classname="tests.test_multi_tenant_integration.TestMultiTenantIntegration" name="test_full_tenant_lifecycle_create_to_delete" time="349.016" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantNamespaceDiscovery" name="test_namespace_qualified_collision_prevention" time="204.729" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantWebhookValidation" name="test_maassubscription_rejected_without_tenant_config_cr" time="7.052" /><testcase classname="tests.test_multi_tenant_integration.TestMultiTenantIntegration" name="test_tenant_namespace_label_change_triggers_reconciliation" time="122.981" /><testcase classname="tests.test_tenant_namespace_discovery.TestTenantWebhookValidation" name="test_maasauthpolicy_rejected_without_tenant_config_cr" time="7.959" /><testcase classname="tests.test_tenant_discovery_isolation" name="test_tenant_discovery_same_tenant_access" time="107.215"><skipped type="pytest.skip" message="Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.</skipped></testcase><testcase classname="tests.test_tenant_discovery_isolation" name="test_tenant_discovery_cross_tenant_isolation" time="0.001"><skipped type="pytest.skip" message="Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.</skipped></testcase><testcase classname="tests.test_tenant_discovery_isolation" name="test_tenant_discovery_unauthorized_access" time="129.136" /><testcase classname="tests.test_gateway_scoped_authpolicy.TestGatewayAuthPolicyLifecycle" name="test_gateway_auth_rego_is_fixed_size" time="3.691" /><testcase classname="tests.test_multi_tenant_maas_api.TestPerTenantMaaSAPI" name="test_tenant_name_environment_variable_set" time="128.407" /><testcase classname="tests.test_multi_tenant_maas_api.TestPerTenantMaaSAPI" name="test_service_routing_isolation" time="151.485" /><testcase classname="tests.test_multi_tenant_integration.TestMultiTenantIntegration" name="test_same_named_resources_across_tenants" time="204.946" /><testcase classname="tests.test_multi_tenant_maas_api.TestPerTenantMaaSAPI" name="test_httproute_tenant_attachment" time="144.729" /><testcase classname="tests.test_multi_tenant_maas_api.TestPerTenantMaaSAPI" name="test_default_and_multiple_tenants_coexist" time="146.303" /><testcase classname="tests.test_multi_tenant_maas_api.TestPerTenantMaaSAPI" name="test_aitenant_creates_dedicated_maas_api_infrastructure" time="307.302" /><testcase classname="tests.test_tenant_discovery_isolation" name="test_tenant_discovery_each_tenant_returns_own_gateway" time="158.462"><skipped type="pytest.skip" message="Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.</skipped></testcase><testcase classname="tests.test_multi_tenant_integration.TestMultiTenantIntegration" name="test_default_tenant_unaffected_by_multitenancy_enablement" time="11.467" /><testcase classname="tests.test_tenant_model_inference.TestTenantModelInference" name="test_model_routes_through_tenant_gateway" time="299.131" /><testcase classname="tests.test_tenant_model_inference.TestTenantModelInference" name="test_inference_succeeds_through_tenant_gateway[tenant_a]" time="127.822" /><testcase classname="tests.test_tenant_model_inference.TestTenantModelInference" name="test_auth_envoyfilters_select_their_gateway_workloads" time="385.971" /><testcase classname="tests.test_tenant_model_inference.TestTenantBodyRouting" name="test_model_in_body_routing[missing-model]" time="148.446" /><testcase classname="tests.test_tenant_model_inference.TestTenantModelInference" name="test_tenant_isolation_cross_gateway_blocked" time="426.976" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_api_key_creation_scoped_to_tenant" time="392.920" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_api_key_validates_against_correct_tenant" time="46.123" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_api_key_list_scoped_to_tenant" time="196.665" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_api_key_subscription_selection_uses_tenant_namespace" time="37.376" /><testcase classname="tests.test_tenant_model_inference.TestTenantBodyRouting" name="test_model_in_body_routing[wrong-model]" time="530.953" /><testcase classname="tests.test_tenant_model_inference.TestTenantBodyRouting" name="test_each_tenant_routes_to_own_model" time="139.855" /><testcase classname="tests.test_tenant_model_inference.TestTenantBodyRouting" name="test_model_in_body_routing[correct-model]" time="521.525" /><testcase classname="tests.test_tenant_model_inference.TestTenantModelInference" name="test_inference_succeeds_through_tenant_gateway[tenant_b]" time="478.844" /><testcase classname="tests.test_tenant_subscription_isolation.TestTenantSubscriptionIsolation" name="test_subscription_selection_per_tenant" time="241.839" /><testcase classname="tests.test_tenant_rate_limit_isolation.TestTenantRateLimitIsolation" name="test_rate_limit_enforced_per_tenant" time="206.657" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_api_key_rejected_cross_tenant" time="235.620" /><testcase classname="tests.test_tenant_rate_limit_isolation.TestTenantRateLimitIsolation" name="test_independent_tenant_rate_limits" time="246.928" /><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_default_tenant_keeps_legacy_ipp_names" time="0.275" /><testcase classname="tests.test_tenant_subscription_isolation.TestTenantSubscriptionIsolation" name="test_subscription_list_scoped_to_tenant" time="358.446" /><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_per_tenant_ipp_deployments_exist" time="264.201" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_oidc_token_validation_per_tenant" time="253.932"><skipped type="pytest.skip" message="Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B">/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B</skipped></testcase><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_per_tenant_ipp_env_vars" time="191.824"><skipped type="pytest.skip" message="praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable">/workspace/source/test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable</skipped></testcase><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPRouting" name="test_default_gateway_hits_default_ipp_only" time="237.937" /><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPRouting" name="test_tenant_gateway_hits_tenant_ipp_only" time="383.621" /><testcase classname="tests.test_tenant_auth_isolation.TestTenantAuthIsolation" name="test_api_key_metadata_not_leaked_cross_tenant" time="390.297" /><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_multiple_tenant_ipp_stacks_coexist" time="403.827"><error message="failed on setup with &quot;AssertionError: aitenant/e2e-ipp-a-a6b650 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{&quot;apiVersion&quot;:&quot;maas.opendatahub.io/v1alpha1&quot;,&quot;kind&quot;:&quot;AITenant&quot;,&quot;metadata&quot;:{&quot;annotations&quot;:{},&quot;name&quot;:&quot;e2e-ipp-a-a6b650&quot;,&quot;namespace&quot;:&quot;ai-tenants&quot;},&quot;spec&quot;:{&quot;gateway&quot;:{&quot;name&quot;:&quot;e2e-ipp-a-a6b650&quot;}}}\n'}, 'creationTimestamp': '2026-10-30T01:...&#10;[truncated; full report in element text]">@pytest.fixture(scope="module")
    def ipp_tenant_cases():
        require_tenant_namespace_discovery()
        require_aitenant_crd()
        case_a = new_named_tenant_case("e2e-ipp-a")
        case_b = new_named_tenant_case("e2e-ipp-b")
        try:
            for case in (case_a, case_b):
&gt;               bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP)

test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant
    wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

kind = 'aitenant', name = 'e2e-ipp-a-a6b650', namespace = 'ai-tenants'
predicate = &lt;function aitenant_ready at 0x7fb7b16d2160&gt;, timeout = 180
interval = 5

    def wait_for_json(
        kind: str,
        name: str,
        namespace: Optional[str] = None,
        *,
        predicate=None,
        timeout: int = 180,
        interval: int = 5,
    ) -&gt; dict:
        deadline = time.time() + timeout
        last_obj = None
        while time.time() &lt; deadline:
            obj = get_json_or_none(kind, name, namespace)
            if obj is not None:
                last_obj = obj
                if predicate is None or predicate(obj):
                    return obj
            time.sleep(interval)
&gt;       raise AssertionError(
            f"{kind}/{name} in {namespace or '&lt;cluster&gt;'} did not satisfy condition. Last object: {last_obj}"
        )
E       AssertionError: aitenant/e2e-ipp-a-a6b650 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-a-a6b650","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-a-a6b650"}}}\n'}, 'creationTimestamp': '2026-10-30T01:24:02Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-a-a6b650', 'namespace': 'ai-tenants', 'resourceVersion': '105952', 'uid': '469bc98b-6115-4c10-a28b-83b922ac94d8'}, 'spec': {'gateway': {'name': 'e2e-ipp-a-a6b650'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T01:24:04Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-a-a6b650', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-a-a6b650'}}

test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError</error></testcase><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPCleanup" name="test_ipp_resources_removed_on_aitenant_delete" time="440.431"><error message="failed on setup with &quot;AssertionError: aitenant/e2e-ipp-a-73831d in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{&quot;apiVersion&quot;:&quot;maas.opendatahub.io/v1alpha1&quot;,&quot;kind&quot;:&quot;AITenant&quot;,&quot;metadata&quot;:{&quot;annotations&quot;:{},&quot;name&quot;:&quot;e2e-ipp-a-73831d&quot;,&quot;namespace&quot;:&quot;ai-tenants&quot;},&quot;spec&quot;:{&quot;gateway&quot;:{&quot;name&quot;:&quot;e2e-ipp-a-73831d&quot;}}}\n'}, 'creationTimestamp': '2026-10-30T01:...&#10;[truncated; full report in element text]">@pytest.fixture(scope="module")
    def ipp_tenant_cases():
        require_tenant_namespace_discovery()
        require_aitenant_crd()
        case_a = new_named_tenant_case("e2e-ipp-a")
        case_b = new_named_tenant_case("e2e-ipp-b")
        try:
            for case in (case_a, case_b):
&gt;               bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP)

test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant
    wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

kind = 'aitenant', name = 'e2e-ipp-a-73831d', namespace = 'ai-tenants'
predicate = &lt;function aitenant_ready at 0x7f8dec686f20&gt;, timeout = 180
interval = 5

    def wait_for_json(
        kind: str,
        name: str,
        namespace: Optional[str] = None,
        *,
        predicate=None,
        timeout: int = 180,
        interval: int = 5,
    ) -&gt; dict:
        deadline = time.time() + timeout
        last_obj = None
        while time.time() &lt; deadline:
            obj = get_json_or_none(kind, name, namespace)
            if obj is not None:
                last_obj = obj
                if predicate is None or predicate(obj):
                    return obj
            time.sleep(interval)
&gt;       raise AssertionError(
            f"{kind}/{name} in {namespace or '&lt;cluster&gt;'} did not satisfy condition. Last object: {last_obj}"
        )
E       AssertionError: aitenant/e2e-ipp-a-73831d in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-a-73831d","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-a-73831d"}}}\n'}, 'creationTimestamp': '2026-10-30T01:23:18Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-a-73831d', 'namespace': 'ai-tenants', 'resourceVersion': '103396', 'uid': '0ce7ab67-14d6-46d0-855b-daadbcc30ca6'}, 'spec': {'gateway': {'name': 'e2e-ipp-a-73831d'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T01:23:19Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-a-73831d', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-a-73831d'}}

test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError</error></testcase><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_per_tenant_envoyfilter_grpc_clusters" time="445.198"><error message="failed on setup with &quot;AssertionError: aitenant/e2e-ipp-a-d3ed60 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{&quot;apiVersion&quot;:&quot;maas.opendatahub.io/v1alpha1&quot;,&quot;kind&quot;:&quot;AITenant&quot;,&quot;metadata&quot;:{&quot;annotations&quot;:{},&quot;name&quot;:&quot;e2e-ipp-a-d3ed60&quot;,&quot;namespace&quot;:&quot;ai-tenants&quot;},&quot;spec&quot;:{&quot;gateway&quot;:{&quot;name&quot;:&quot;e2e-ipp-a-d3ed60&quot;}}}\n'}, 'creationTimestamp': '2026-10-30T01:...&#10;[truncated; full report in element text]">@pytest.fixture(scope="module")
    def ipp_tenant_cases():
        require_tenant_namespace_discovery()
        require_aitenant_crd()
        case_a = new_named_tenant_case("e2e-ipp-a")
        case_b = new_named_tenant_case("e2e-ipp-b")
        try:
            for case in (case_a, case_b):
&gt;               bootstrap_aitenant_tenant(case, payload_processing_type=PAYLOAD_PROCESSING_TYPE_IPP)

test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:124: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:1047: in bootstrap_aitenant_tenant
    wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, predicate=aitenant_ready)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

kind = 'aitenant', name = 'e2e-ipp-a-d3ed60', namespace = 'ai-tenants'
predicate = &lt;function aitenant_ready at 0x7f11460daf20&gt;, timeout = 180
interval = 5

    def wait_for_json(
        kind: str,
        name: str,
        namespace: Optional[str] = None,
        *,
        predicate=None,
        timeout: int = 180,
        interval: int = 5,
    ) -&gt; dict:
        deadline = time.time() + timeout
        last_obj = None
        while time.time() &lt; deadline:
            obj = get_json_or_none(kind, name, namespace)
            if obj is not None:
                last_obj = obj
                if predicate is None or predicate(obj):
                    return obj
            time.sleep(interval)
&gt;       raise AssertionError(
            f"{kind}/{name} in {namespace or '&lt;cluster&gt;'} did not satisfy condition. Last object: {last_obj}"
        )
E       AssertionError: aitenant/e2e-ipp-a-d3ed60 in ai-tenants did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'AITenant', 'metadata': {'annotations': {'kubectl.kubernetes.io/last-applied-configuration': '{"apiVersion":"maas.opendatahub.io/v1alpha1","kind":"AITenant","metadata":{"annotations":{},"name":"e2e-ipp-a-d3ed60","namespace":"ai-tenants"},"spec":{"gateway":{"name":"e2e-ipp-a-d3ed60"}}}\n'}, 'creationTimestamp': '2026-10-30T01:23:23Z', 'finalizers': ['maas.opendatahub.io/aitenant-cleanup'], 'generation': 1, 'name': 'e2e-ipp-a-d3ed60', 'namespace': 'ai-tenants', 'resourceVersion': '103742', 'uid': '03ab6d36-8e66-46d5-b9a5-f0cd4997373d'}, 'spec': {'gateway': {'name': 'e2e-ipp-a-d3ed60'}}, 'status': {'conditions': [{'lastTransitionTime': '2026-10-30T01:23:25Z', 'message': 'waiting for MaasTenantConfig to report Ready', 'observedGeneration': 1, 'reason': 'TenantConfigNotReady', 'status': 'False', 'type': 'Ready'}], 'gatewayRef': {'name': 'e2e-ipp-a-d3ed60', 'namespace': 'openshift-ingress'}, 'phase': 'Pending', 'tenantNamespace': 'ai-tenant-e2e-ipp-a-d3ed60'}}

test/maas-e2e/test/e2e/tests/multitenancy_helpers.py:268: AssertionError</error></testcase><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_per_tenant_envoyfilter_workload_selector_isolated" time="476.305" /><testcase classname="tests.test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure" name="test_per_tenant_networkpolicy_when_applied" time="380.077" /></testsuite></testsuites>