/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:941: no Kuadrant WasmPlugin for gateway maas-default-gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1728: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestManagedAnnotation object at 0x7f10bdbcc790> @pytest.mark.serial def test_trlp_managed_false_prevents_update(self): """TokenRateLimitPolicy annotated with opendatahub.io/managed=false must not have its spec updated when the parent MaaSSubscription is modified.""" ns = _ns() trlp_ns = MODEL_NAMESPACE parent_snapshot = None try: # 1. Verify the TRLP exists trlp = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp, f"TokenRateLimitPolicy {TRLP_NAME} not found in {trlp_ns}" # 2. Snapshot the parent MaaSSubscription for cleanup parent_snapshot = _snapshot_cr( "maassubscription", SIMULATOR_SUBSCRIPTION, ns ) assert parent_snapshot, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} not found in {ns}" ) # 3. Annotate the TRLP with managed=false _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}=false", trlp_ns, ) log.info( "Annotated TokenRateLimitPolicy %s with %s=false", TRLP_NAME, MANAGED_ANNOTATION, ) # 4. Re-read the TRLP to capture baseline spec (post-annotation) trlp_baseline = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_baseline, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after annotation" ) baseline_spec = trlp_baseline["spec"] # 5. Modify the parent MaaSSubscription (change the token rate limit value) modified_parent = copy.deepcopy(parent_snapshot) model_refs = modified_parent["spec"].get("modelRefs", []) assert model_refs, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} has no modelRefs" ) for ref in model_refs: if ref.get("name") == MODEL_REF: limits = ref.get("tokenRateLimits", []) assert limits, f"modelRef {MODEL_REF} has no tokenRateLimits" limits[0]["limit"] = limits[0]["limit"] + 99999 break else: pytest.fail( f"modelRef {MODEL_REF} not found in MaaSSubscription {SIMULATOR_SUBSCRIPTION}" ) _apply_cr(modified_parent) log.info( "Modified parent MaaSSubscription %s (changed token rate limit)", SIMULATOR_SUBSCRIPTION, ) # 6. Wait for the controller to reconcile the edited spec modified = _get_cr("maassubscription", SIMULATOR_SUBSCRIPTION, ns) assert modified, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} disappeared after update" ) reconciled = _wait_for_subscription_generation_observed( SIMULATOR_SUBSCRIPTION, modified["metadata"]["generation"], namespace=ns, ) # An unchanged TRLP only proves managed=false if the reconcile got # to the TRLP step: the model was valid and the step did not fail. # Degraded is expected while Kuadrant has not enforced the policy. status = reconciled.get("status", {}) model_ready = any( s.get("name") == MODEL_REF and s.get("ready") is True for s in status.get("modelRefStatuses", []) ) assert model_ready and status.get("phase") in ("Active", "Degraded"), ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} reconciled without " f"reaching the TRLP step: {status}" ) # 7. Re-read the TRLP and compare spec trlp_after = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_after, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after parent update" ) after_spec = trlp_after["spec"] assert baseline_spec == after_spec, ( f"TokenRateLimitPolicy spec changed despite {MANAGED_ANNOTATION}=false.\n" f"Before: {json.dumps(baseline_spec, indent=2)}\n" f"After: {json.dumps(after_spec, indent=2)}" ) log.info( "TokenRateLimitPolicy spec unchanged after parent modification — managed=false respected" ) finally: # Remove the annotation (best-effort so parent restore still runs) try: _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}-", trlp_ns ) log.info( "Removed %s annotation from TokenRateLimitPolicy %s", MANAGED_ANNOTATION, TRLP_NAME, ) except subprocess.CalledProcessError: log.warning( "Failed to remove %s annotation from TokenRateLimitPolicy %s (may not exist)", MANAGED_ANNOTATION, TRLP_NAME, ) # Restore the parent MaaSSubscription if parent_snapshot: _apply_cr(parent_snapshot) log.info( "Restored parent MaaSSubscription %s from snapshot", SIMULATOR_SUBSCRIPTION, ) > _wait_for_subscription_inference_ready( SIMULATOR_SUBSCRIPTION, MODEL_REF, model_namespace=MODEL_NAMESPACE, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1938: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1833: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'facebook-opt-125m-simulated', model_namespace = 'llm' timeout = 300, subscription_name = 'simulator-subscription' subscription_namespace = 'models-as-a-service' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=180, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 180) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-facebook-opt-125m-simulated was not created and enforced in llm within 300s: generation=1, observedGeneration=1, Enforced={'lastTransitionTime': '2026-11-03T20:32:49Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1575: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant.py:401: default payload-processing is praxis-extproc; MaasTenantConfig.spec.payloadProcessing.resources is MaaS IPP-only/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:359: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueself = <test_multi_tenant_maas_api.TestPerTenantMaaSAPI object at 0x7f10bda67890> dual_listener_tenant_case = {'suffix': 'ca53c0', 'tenant_ns': 'ai-tenant-e2e-authcfg-http-ca53c0', 'tenant_label_name': 'e2e-authcfg-http-ca53c0', 'gateway_name': 'e2e-authcfg-http-ca53c0', ...} @pytest.mark.nightly @pytest.mark.serial def test_dual_listeners_keep_one_authconfig_per_route_rule(self, dual_listener_tenant_case): """One stable Authorino AuthConfig per rule when both listeners attach the route.""" case = dual_listener_tenant_case gateway = wait_for_json( "gateway", case["gateway_name"], GATEWAY_NAMESPACE, predicate=lambda obj: {"http", "https"} <= _gateway_listener_names_with_routes(obj), timeout=180, ) listeners = { listener.get("name"): listener for listener in (gateway.get("spec") or {}).get("listeners") or [] } assert {"http", "https"} <= set(listeners), ( f"{case['gateway_name']} must have both HTTP and HTTPS listeners, got {listeners!r}" ) assert listeners["http"].get("protocol") == "HTTP" and listeners["http"].get("port") == 80 assert listeners["https"].get("protocol") == "HTTPS" and listeners["https"].get("port") == 443 assert listeners["http"].get("hostname") == listeners["https"].get("hostname"), ( f"Test setup must use a shared hostname across listeners: {listeners!r}" ) route_name = _tenant_api_names(case)["httproute"] route = wait_for_json( "httproute", route_name, INFRA_NAMESPACE, predicate=lambda obj: _route_accepted_by_gateway(obj, case["gateway_name"]), timeout=180, ) authconfigs_by_rule = _wait_for_stable_route_authconfigs(route) rules = (route.get("spec") or {}).get("rules") or [] expected_rules = {f"rule-{index}" for index in range(1, len(rules) + 1)} assert set(authconfigs_by_rule) == expected_rules > assert all( len(observations) == 1 and observations[0][2] for observations in authconfigs_by_rule.values() ), ( f"Expected exactly one Ready and Available non-deleting Authorino " f"AuthConfig per rule for " f"{INFRA_NAMESPACE}/{route_name}; got {authconfigs_by_rule!r}" ) E AssertionError: Expected exactly one Ready and Available non-deleting Authorino AuthConfig per rule for odh-ai-gateway-infra/maas-api-route-e2e-authcfg-http-ca53c0; got {'rule-1': [('0ebe243155eb06a498aa2cea656a22cd05f5d7fa1636862a13ec0f121486ff95', '9928fe29-d35f-40b3-8c31-44786db661f0', True), ('795de51b62d0cc6eaca21da870d0f38d52d61c99f35ecbe409eb4e4415a3676a', '7ef8fcdf-075f-4ca5-9103-0d5fb80755f4', True)], 'rule-6': [('4846b3b38ce49d784b8d9ddcd145ff1ee55bcf6ff54548a3cae3b66267989ccf', '96dfee3e-0b4f-404f-9728-4ef35ba7625b', True), ('9acfeae5d2002312f9c4f7ad5693274090543495abf4ddbbd9434558de7ff1a0', '407438c3-b9ca-4d18-98a7-b25ba3d083cc', True)], 'rule-4': [('4ec82168f6a658263eabf50635ec8ab1ef7876f8e76184743ba78f54b5bff8d5', '053d5d84-37dc-4eb4-aa4c-bf4c9d703401', True), ('ba37608d04fc9bd7efd58b2934e405fdae785f60b5fe88e760b8d6b687705101', '0333f1b2-631b-48d6-8cdd-81682f6fb0cf', True)], 'rule-3': [('6618dab7ef096e5f58515b8ba07b60082f4b056bafe574f7a96675b38d7441e0', 'd4ac13d8-35a9-4c0e-8707-d8f409f1c3bb', True), ('aad16bfe68e727577878b7c873990697d8c1f1c242cb2f319768a369cf1d55d0', 'df9d8587-fdf3-4554-bc13-9283326ebe3d', True)], 'rule-2': [('d27a5acc09248405a796d6bcfa54a74f11be4f8d30944ce6afcc66aa9fd9febd', '862c5ba7-b2f0-4fed-81f9-458cb7810dc8', True), ('e0b0e3c99af97488c9da9fedf4103a6bb2634a0139bc999bd7de8c14e3189aae', 'bd30c222-6dae-4bb1-8f7b-35c0d73db08d', True)], 'rule-5': [('da1c2d148eb2bb56f7029b987596718d0fa6f0852dfef8c65c0730adda17f49a', 'c77b71cf-6f18-4219-81ae-b606b76bf34b', True), ('fdebcca622ec949975f8ca31ebcf25c2e9c66afd23ea4eb22723d3ad32645a63', '4f1897fe-2d51-4f3a-8259-1ecbc54ff71f', True)]} E assert False E + where False = all(<generator object TestPerTenantMaaSAPI.test_dual_listeners_keep_one_authconfig_per_route_rule.<locals>.<genexpr> at 0x7f10bdc958c0>) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:381: AssertionError