/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_subscription.TestManagedAnnotation object at 0x7f52698b78d0> def test_trlp_managed_false_prevents_update(self): """TokenRateLimitPolicy annotated with opendatahub.io/managed=false must not have its spec updated when the parent MaaSSubscription is modified.""" ns = _ns() trlp_ns = MODEL_NAMESPACE parent_snapshot = None try: # 1. Verify the TRLP exists trlp = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp, f"TokenRateLimitPolicy {TRLP_NAME} not found in {trlp_ns}" # 2. Snapshot the parent MaaSSubscription for cleanup parent_snapshot = _snapshot_cr( "maassubscription", SIMULATOR_SUBSCRIPTION, ns ) assert parent_snapshot, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} not found in {ns}" ) # 3. Annotate the TRLP with managed=false _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}=false", trlp_ns, ) log.info( "Annotated TokenRateLimitPolicy %s with %s=false", TRLP_NAME, MANAGED_ANNOTATION, ) # 4. Re-read the TRLP to capture baseline spec (post-annotation) trlp_baseline = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_baseline, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after annotation" ) baseline_spec = trlp_baseline["spec"] # 5. Modify the parent MaaSSubscription (change the token rate limit value) modified_parent = copy.deepcopy(parent_snapshot) model_refs = modified_parent["spec"].get("modelRefs", []) assert model_refs, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} has no modelRefs" ) for ref in model_refs: if ref.get("name") == MODEL_REF: limits = ref.get("tokenRateLimits", []) assert limits, f"modelRef {MODEL_REF} has no tokenRateLimits" limits[0]["limit"] = limits[0]["limit"] + 99999 break else: pytest.fail( f"modelRef {MODEL_REF} not found in MaaSSubscription {SIMULATOR_SUBSCRIPTION}" ) _apply_cr(modified_parent) log.info( "Modified parent MaaSSubscription %s (changed token rate limit)", SIMULATOR_SUBSCRIPTION, ) # 6. Wait for the controller to reconcile the edited spec modified = _get_cr("maassubscription", SIMULATOR_SUBSCRIPTION, ns) assert modified, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} disappeared after update" ) reconciled = _wait_for_subscription_generation_observed( SIMULATOR_SUBSCRIPTION, modified["metadata"]["generation"], namespace=ns, ) # An unchanged TRLP only proves managed=false if the reconcile got # to the TRLP step: the model was valid and the step did not fail. # Degraded is expected while Kuadrant has not enforced the policy. status = reconciled.get("status", {}) model_ready = any( s.get("name") == MODEL_REF and s.get("ready") is True for s in status.get("modelRefStatuses", []) ) assert model_ready and status.get("phase") in ("Active", "Degraded"), ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} reconciled without " f"reaching the TRLP step: {status}" ) # 7. Re-read the TRLP and compare spec trlp_after = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_after, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after parent update" ) after_spec = trlp_after["spec"] assert baseline_spec == after_spec, ( f"TokenRateLimitPolicy spec changed despite {MANAGED_ANNOTATION}=false.\n" f"Before: {json.dumps(baseline_spec, indent=2)}\n" f"After: {json.dumps(after_spec, indent=2)}" ) log.info( "TokenRateLimitPolicy spec unchanged after parent modification — managed=false respected" ) finally: # Remove the annotation (best-effort so parent restore still runs) try: _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}-", trlp_ns ) log.info( "Removed %s annotation from TokenRateLimitPolicy %s", MANAGED_ANNOTATION, TRLP_NAME, ) except subprocess.CalledProcessError: log.warning( "Failed to remove %s annotation from TokenRateLimitPolicy %s (may not exist)", MANAGED_ANNOTATION, TRLP_NAME, ) # Restore the parent MaaSSubscription if parent_snapshot: _apply_cr(parent_snapshot) log.info( "Restored parent MaaSSubscription %s from snapshot", SIMULATOR_SUBSCRIPTION, ) > _wait_for_subscription_inference_ready( SIMULATOR_SUBSCRIPTION, MODEL_REF, model_namespace=MODEL_NAMESPACE, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1789: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w5-6f4fba' model_namespace = 'e2e-models-e2e-worker-w5-6f4fba', timeout = 300 subscription_name = 'simulator-subscription' subscription_namespace = 'ai-tenant-e2e-worker-w5-6f4fba' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w5-6f4fba was not created and enforced in e2e-models-e2e-worker-w5-6f4fba within 300s: generation=3, observedGeneration=3, Enforced={'lastTransitionTime': '2026-10-30T00:45:00Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7fbd8896fed0> def test_e2e_with_both_access_and_subscription_gets_200(self): """ API key with both access and subscription should get 200 OK. Uses an existing model reference that aligns with gateway path-based model identity. """ ns = _ns() auth_policy_name = "e2e-test-auth-success" subscription_name = "e2e-test-subscription-success" sa_name = "e2e-sa-success" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Use pre-existing premium model to keep model identity consistent with gateway path. _create_test_auth_policy(auth_policy_name, PREMIUM_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, PREMIUM_MODEL_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth_policy_name, namespace=ns, timeout=120, require_auth_policies=False) > _wait_for_subscription_inference_ready( subscription_name, PREMIUM_MODEL_REF, namespace=ns, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1887: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'premium-w1-29be73' model_namespace = 'e2e-models-e2e-worker-w1-29be73', timeout = 180 subscription_name = 'e2e-test-subscription-success' subscription_namespace = 'ai-tenant-e2e-worker-w1-29be73' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-premium-w1-29be73 was not created and enforced in e2e-models-e2e-worker-w1-29be73 within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T00:45:00Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7fdad71a6a90> def test_e2e_group_based_access_gets_200(self): """ E2E test: Group-based auth and subscription (success case). Validates that users can access models via group membership in both MaaSAuthPolicy and MaaSSubscription, not just explicit user lists. """ ns = _ns() auth_policy_name = "e2e-test-group-auth" subscription_name = "e2e-test-group-subscription" sa_name = "e2e-sa-group" # Use namespace-specific group that SA will be in test_group = f"system:serviceaccounts:{ns}" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) # Create auth policy using GROUP (not user) _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group]) # Create subscription using GROUP (not user) _create_test_subscription(subscription_name, MODEL_REF, groups=[test_group]) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_name, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2216: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w3-cefb27' model_namespace = 'e2e-models-e2e-worker-w3-cefb27', timeout = 180 subscription_name = 'e2e-test-group-subscription' subscription_namespace = 'ai-tenant-e2e-worker-w3-cefb27' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w3-cefb27 was not created and enforced in e2e-models-e2e-worker-w3-cefb27 within 180s: generation=6, observedGeneration=6, Enforced={'lastTransitionTime': '2026-10-30T00:44:17Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1580: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.self = <test_subscription.TestE2ESubscriptionFlow object at 0x7f52698e5c50> def test_e2e_multiple_subscriptions_separate_keys_gets_200(self): """ User with two subscriptions for the same model: mint one API key per subscription; each key succeeds on inference without x-maas-subscription. """ ns = _ns() auth_policy_name = "e2e-test-auth-multi-sub-valid" subscription_1 = "e2e-test-subscription-free" subscription_2 = "e2e-test-subscription-premium" sa_name = "e2e-sa-multi-sub-valid" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources with 2 subscriptions for the same user _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_1, MODEL_REF, users=[sa_user], token_limit=100) _create_test_subscription(subscription_2, MODEL_REF, users=[sa_user], token_limit=1000) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_1, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2089: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w5-6f4fba' model_namespace = 'e2e-models-e2e-worker-w5-6f4fba', timeout = 180 subscription_name = 'e2e-test-subscription-free' subscription_namespace = 'ai-tenant-e2e-worker-w5-6f4fba' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w5-6f4fba was not created and enforced in e2e-models-e2e-worker-w5-6f4fba within 180s: generation=5, observedGeneration=5, Enforced={'lastTransitionTime': '2026-10-30T00:45:00Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_Bself = <test_tenant_rate_limit_isolation.TestTenantRateLimitIsolation object at 0x7f59a76954d0> tenant_rate_limit_setup = {'tenant_a': {'suffix': '009e78', 'tenant_ns': 'ai-tenant-e2e-shared-a-w6-009e78', 'tenant_label_name': 'e2e-shared-a-...: 'e2e-shared-b-w6-7f92f6', ...}, 'policy': 'e2e-rate-iso-auth-b0ce1a', 'subscription_a': 'e2e-rate-iso-a-b0ce1a', ...} def test_independent_tenant_rate_limits(self, tenant_rate_limit_setup): """5.2: Exhausting Tenant A does not consume Tenant B's quota.""" tenant_a = tenant_rate_limit_setup["tenant_a"] tenant_b = tenant_rate_limit_setup["tenant_b"] successes, response = _exhaust_until_429( tenant_a["base_url"], tenant_rate_limit_setup["key_a"], tenant_a["model_path"], tenant_a["backend_model_name"], ) assert successes > 0, f"Tenant A hit 429 before any successful inference: {response_summary(response)}" > assert response.status_code == 429, ( f"Tenant A did not hit rate limit after {successes} successes: {response_summary(response)}" ) E AssertionError: Tenant A did not hit rate limit after 41 successes: status=200 x-ext-auth-reason=None body={"id":"cmpl-baa50482-d136-5041-92ec-89d1a9a87bee","created":1793323011,"model":"facebook/opt-125m","usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2},"object":"text_completion","kv_transfer_params":null,"choices":[{"index":0,"finish_reason":"length","text":"To "}]} E assert 200 == 429 E + where 200 = <Response [200]>.status_code test/maas-e2e/test/e2e/tests/test_tenant_rate_limit_isolation.py:249: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)