/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:941: no Kuadrant WasmPlugin for gateway maas-default-gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1728: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestManagedAnnotation object at 0x7f9591ddd090> @pytest.mark.serial def test_trlp_managed_false_prevents_update(self): """TokenRateLimitPolicy annotated with opendatahub.io/managed=false must not have its spec updated when the parent MaaSSubscription is modified.""" ns = _ns() trlp_ns = MODEL_NAMESPACE parent_snapshot = None try: # 1. Verify the TRLP exists trlp = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp, f"TokenRateLimitPolicy {TRLP_NAME} not found in {trlp_ns}" # 2. Snapshot the parent MaaSSubscription for cleanup parent_snapshot = _snapshot_cr( "maassubscription", SIMULATOR_SUBSCRIPTION, ns ) assert parent_snapshot, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} not found in {ns}" ) # 3. Annotate the TRLP with managed=false _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}=false", trlp_ns, ) log.info( "Annotated TokenRateLimitPolicy %s with %s=false", TRLP_NAME, MANAGED_ANNOTATION, ) # 4. Re-read the TRLP to capture baseline spec (post-annotation) trlp_baseline = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_baseline, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after annotation" ) baseline_spec = trlp_baseline["spec"] # 5. Modify the parent MaaSSubscription (change the token rate limit value) modified_parent = copy.deepcopy(parent_snapshot) model_refs = modified_parent["spec"].get("modelRefs", []) assert model_refs, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} has no modelRefs" ) for ref in model_refs: if ref.get("name") == MODEL_REF: limits = ref.get("tokenRateLimits", []) assert limits, f"modelRef {MODEL_REF} has no tokenRateLimits" limits[0]["limit"] = limits[0]["limit"] + 99999 break else: pytest.fail( f"modelRef {MODEL_REF} not found in MaaSSubscription {SIMULATOR_SUBSCRIPTION}" ) _apply_cr(modified_parent) log.info( "Modified parent MaaSSubscription %s (changed token rate limit)", SIMULATOR_SUBSCRIPTION, ) # 6. Wait for the controller to reconcile the edited spec modified = _get_cr("maassubscription", SIMULATOR_SUBSCRIPTION, ns) assert modified, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} disappeared after update" ) reconciled = _wait_for_subscription_generation_observed( SIMULATOR_SUBSCRIPTION, modified["metadata"]["generation"], namespace=ns, ) # An unchanged TRLP only proves managed=false if the reconcile got # to the TRLP step: the model was valid and the step did not fail. # Degraded is expected while Kuadrant has not enforced the policy. status = reconciled.get("status", {}) model_ready = any( s.get("name") == MODEL_REF and s.get("ready") is True for s in status.get("modelRefStatuses", []) ) assert model_ready and status.get("phase") in ("Active", "Degraded"), ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} reconciled without " f"reaching the TRLP step: {status}" ) # 7. Re-read the TRLP and compare spec trlp_after = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_after, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after parent update" ) after_spec = trlp_after["spec"] assert baseline_spec == after_spec, ( f"TokenRateLimitPolicy spec changed despite {MANAGED_ANNOTATION}=false.\n" f"Before: {json.dumps(baseline_spec, indent=2)}\n" f"After: {json.dumps(after_spec, indent=2)}" ) log.info( "TokenRateLimitPolicy spec unchanged after parent modification — managed=false respected" ) finally: # Remove the annotation (best-effort so parent restore still runs) try: _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}-", trlp_ns ) log.info( "Removed %s annotation from TokenRateLimitPolicy %s", MANAGED_ANNOTATION, TRLP_NAME, ) except subprocess.CalledProcessError: log.warning( "Failed to remove %s annotation from TokenRateLimitPolicy %s (may not exist)", MANAGED_ANNOTATION, TRLP_NAME, ) # Restore the parent MaaSSubscription if parent_snapshot: _apply_cr(parent_snapshot) log.info( "Restored parent MaaSSubscription %s from snapshot", SIMULATOR_SUBSCRIPTION, ) > _wait_for_subscription_inference_ready( SIMULATOR_SUBSCRIPTION, MODEL_REF, model_namespace=MODEL_NAMESPACE, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1938: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1833: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'facebook-opt-125m-simulated', model_namespace = 'llm' timeout = 300, subscription_name = 'simulator-subscription' subscription_namespace = 'models-as-a-service' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=180, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 180) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-facebook-opt-125m-simulated was not created and enforced in llm within 300s: generation=1, observedGeneration=1, Enforced={'lastTransitionTime': '2026-11-01T22:55:55Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1575: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant.py:401: default payload-processing is praxis-extproc; MaasTenantConfig.spec.payloadProcessing.resources is MaaS IPP-only/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:359: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueself = <test_multi_tenant_maas_api.TestPerTenantMaaSAPI object at 0x7f9591c73ed0> dual_listener_tenant_case = {'suffix': '4b45d6', 'tenant_ns': 'ai-tenant-e2e-authcfg-http-4b45d6', 'tenant_label_name': 'e2e-authcfg-http-4b45d6', 'gateway_name': 'e2e-authcfg-http-4b45d6', ...} @pytest.mark.nightly @pytest.mark.serial def test_dual_listeners_keep_one_authconfig_per_route_rule(self, dual_listener_tenant_case): """One stable Authorino AuthConfig per rule when both listeners attach the route.""" case = dual_listener_tenant_case gateway = wait_for_json( "gateway", case["gateway_name"], GATEWAY_NAMESPACE, predicate=lambda obj: {"http", "https"} <= _gateway_listener_names_with_routes(obj), timeout=180, ) listeners = { listener.get("name"): listener for listener in (gateway.get("spec") or {}).get("listeners") or [] } assert {"http", "https"} <= set(listeners), ( f"{case['gateway_name']} must have both HTTP and HTTPS listeners, got {listeners!r}" ) assert listeners["http"].get("protocol") == "HTTP" and listeners["http"].get("port") == 80 assert listeners["https"].get("protocol") == "HTTPS" and listeners["https"].get("port") == 443 assert listeners["http"].get("hostname") == listeners["https"].get("hostname"), ( f"Test setup must use a shared hostname across listeners: {listeners!r}" ) route_name = _tenant_api_names(case)["httproute"] route = wait_for_json( "httproute", route_name, INFRA_NAMESPACE, predicate=lambda obj: _route_accepted_by_gateway(obj, case["gateway_name"]), timeout=180, ) authconfigs_by_rule = _wait_for_stable_route_authconfigs(route) rules = (route.get("spec") or {}).get("rules") or [] expected_rules = {f"rule-{index}" for index in range(1, len(rules) + 1)} assert set(authconfigs_by_rule) == expected_rules > assert all( len(observations) == 1 and observations[0][2] for observations in authconfigs_by_rule.values() ), ( f"Expected exactly one Ready and Available non-deleting Authorino " f"AuthConfig per rule for " f"{INFRA_NAMESPACE}/{route_name}; got {authconfigs_by_rule!r}" ) E AssertionError: Expected exactly one Ready and Available non-deleting Authorino AuthConfig per rule for odh-ai-gateway-infra/maas-api-route-e2e-authcfg-http-4b45d6; got {'rule-6': [('0da9595a52848d3eea4d458d2bc5bc57d6a5484f032776cc20d2a69f7ba29b63', '02c8a4af-502b-44f5-b9c0-7f5d172c26ea', True), ('c71548a139972111631272e8b11ea0011d13186bb03236b02754e6d9633c1f77', '08753a3b-3284-42bf-8dd1-395e9a3f47ad', True)], 'rule-1': [('4feb9fdb6b995e2dbe58293ae7f53fcfe09282c31b37dc90a2d221076c156434', '87448202-dae7-4fcb-867a-d602176801d8', True), ('f487835e6b7759f956996fd9a01c1f289cc2414d597b57cf79c3be71e7195fd7', '6784d4a6-9dbe-467f-a07a-a285a142084a', True)], 'rule-2': [('54339a9d574fdd78a58b1bd4c43b855cc4ffeb12a9322f21dd4e2fc56a6e9c51', '091f3a47-3024-4dde-b7fa-c65650fb91c5', True), ('9b63cf3f69a5e1b7ec066742e1d203ba1637fb87c51adbba1f8b9fa6373bce35', 'd7d37abe-1590-44c8-b4fd-2f2747acbf54', True)], 'rule-4': [('7a00168573d4bdedb5d9a89b00ce49f8e4b8d979e9f869051176d7f1de35f2c4', 'bd9b4748-34d5-4e44-bbe3-9551f856dfa2', True), ('9a8a1c479c90541db6362358fa441a16b3ca8a02e1d6021dcc51154e530eba48', '233a06d0-3a6a-44a0-9f38-65b37eefc9f1', True)], 'rule-3': [('8daf5867aa188f85e1ec777ee57c842a3e3394c13ae66d3e2a04d747716c75bc', 'aac837b6-b658-4ea5-a8e5-c3d64ef3bc18', True), ('f08599c1818dfe2d3b2c141b588615fa5cc02167b4ed2c9db9c5270ef3c6e798', '931a9c78-0527-4548-bccc-69ae95092977', True)], 'rule-5': [('a9892f3c549bff6eda53a680373a79a32aa0cac450487cef0fe4dd832d512740', '1d042e08-782e-446d-a76b-3f7245d7071f', True), ('e4630bc4bfe1bb2b5ce7ed4e2f245a6a83cb7ade562dac5c2bf592194e0a4aa3', 'c52a80d1-2dfb-48ec-9947-397746efc330', True)]} E assert False E + where False = all(<generator object TestPerTenantMaaSAPI.test_dual_listeners_keep_one_authconfig_per_route_rule.<locals>.<genexpr> at 0x7f9591ea58c0>) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:381: AssertionError