/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:941: no Kuadrant WasmPlugin for gateway maas-default-gateway/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1728: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestManagedAnnotation object at 0x7f79114d4550> @pytest.mark.serial def test_trlp_managed_false_prevents_update(self): """TokenRateLimitPolicy annotated with opendatahub.io/managed=false must not have its spec updated when the parent MaaSSubscription is modified.""" ns = _ns() trlp_ns = MODEL_NAMESPACE parent_snapshot = None try: # 1. Verify the TRLP exists trlp = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp, f"TokenRateLimitPolicy {TRLP_NAME} not found in {trlp_ns}" # 2. Snapshot the parent MaaSSubscription for cleanup parent_snapshot = _snapshot_cr( "maassubscription", SIMULATOR_SUBSCRIPTION, ns ) assert parent_snapshot, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} not found in {ns}" ) # 3. Annotate the TRLP with managed=false _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}=false", trlp_ns, ) log.info( "Annotated TokenRateLimitPolicy %s with %s=false", TRLP_NAME, MANAGED_ANNOTATION, ) # 4. Re-read the TRLP to capture baseline spec (post-annotation) trlp_baseline = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_baseline, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after annotation" ) baseline_spec = trlp_baseline["spec"] # 5. Modify the parent MaaSSubscription (change the token rate limit value) modified_parent = copy.deepcopy(parent_snapshot) model_refs = modified_parent["spec"].get("modelRefs", []) assert model_refs, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} has no modelRefs" ) for ref in model_refs: if ref.get("name") == MODEL_REF: limits = ref.get("tokenRateLimits", []) assert limits, f"modelRef {MODEL_REF} has no tokenRateLimits" limits[0]["limit"] = limits[0]["limit"] + 99999 break else: pytest.fail( f"modelRef {MODEL_REF} not found in MaaSSubscription {SIMULATOR_SUBSCRIPTION}" ) _apply_cr(modified_parent) log.info( "Modified parent MaaSSubscription %s (changed token rate limit)", SIMULATOR_SUBSCRIPTION, ) # 6. Wait for the controller to reconcile the edited spec modified = _get_cr("maassubscription", SIMULATOR_SUBSCRIPTION, ns) assert modified, ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} disappeared after update" ) reconciled = _wait_for_subscription_generation_observed( SIMULATOR_SUBSCRIPTION, modified["metadata"]["generation"], namespace=ns, ) # An unchanged TRLP only proves managed=false if the reconcile got # to the TRLP step: the model was valid and the step did not fail. # Degraded is expected while Kuadrant has not enforced the policy. status = reconciled.get("status", {}) model_ready = any( s.get("name") == MODEL_REF and s.get("ready") is True for s in status.get("modelRefStatuses", []) ) assert model_ready and status.get("phase") in ("Active", "Degraded"), ( f"MaaSSubscription {SIMULATOR_SUBSCRIPTION} reconciled without " f"reaching the TRLP step: {status}" ) # 7. Re-read the TRLP and compare spec trlp_after = _get_cr("tokenratelimitpolicy", TRLP_NAME, trlp_ns) assert trlp_after, ( f"TokenRateLimitPolicy {TRLP_NAME} disappeared after parent update" ) after_spec = trlp_after["spec"] assert baseline_spec == after_spec, ( f"TokenRateLimitPolicy spec changed despite {MANAGED_ANNOTATION}=false.\n" f"Before: {json.dumps(baseline_spec, indent=2)}\n" f"After: {json.dumps(after_spec, indent=2)}" ) log.info( "TokenRateLimitPolicy spec unchanged after parent modification — managed=false respected" ) finally: # Remove the annotation (best-effort so parent restore still runs) try: _annotate( "tokenratelimitpolicy", TRLP_NAME, f"{MANAGED_ANNOTATION}-", trlp_ns ) log.info( "Removed %s annotation from TokenRateLimitPolicy %s", MANAGED_ANNOTATION, TRLP_NAME, ) except subprocess.CalledProcessError: log.warning( "Failed to remove %s annotation from TokenRateLimitPolicy %s (may not exist)", MANAGED_ANNOTATION, TRLP_NAME, ) # Restore the parent MaaSSubscription if parent_snapshot: _apply_cr(parent_snapshot) log.info( "Restored parent MaaSSubscription %s from snapshot", SIMULATOR_SUBSCRIPTION, ) > _wait_for_subscription_inference_ready( SIMULATOR_SUBSCRIPTION, MODEL_REF, model_namespace=MODEL_NAMESPACE, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1938: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1833: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'facebook-opt-125m-simulated', model_namespace = 'llm' timeout = 300, subscription_name = 'simulator-subscription' subscription_namespace = 'models-as-a-service' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=180, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 180) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-facebook-opt-125m-simulated was not created and enforced in llm within 300s: generation=1, observedGeneration=1, Enforced={'lastTransitionTime': '2026-11-03T16:27:56Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1575: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant.py:401: default payload-processing is praxis-extproc; MaasTenantConfig.spec.payloadProcessing.resources is MaaS IPP-only/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:359: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=trueself = <test_multi_tenant_maas_api.TestPerTenantMaaSAPI object at 0x7f791136f8d0> dual_listener_tenant_case = {'suffix': '05bcf4', 'tenant_ns': 'ai-tenant-e2e-authcfg-http-05bcf4', 'tenant_label_name': 'e2e-authcfg-http-05bcf4', 'gateway_name': 'e2e-authcfg-http-05bcf4', ...} @pytest.mark.nightly @pytest.mark.serial def test_dual_listeners_keep_one_authconfig_per_route_rule(self, dual_listener_tenant_case): """One stable Authorino AuthConfig per rule when both listeners attach the route.""" case = dual_listener_tenant_case gateway = wait_for_json( "gateway", case["gateway_name"], GATEWAY_NAMESPACE, predicate=lambda obj: {"http", "https"} <= _gateway_listener_names_with_routes(obj), timeout=180, ) listeners = { listener.get("name"): listener for listener in (gateway.get("spec") or {}).get("listeners") or [] } assert {"http", "https"} <= set(listeners), ( f"{case['gateway_name']} must have both HTTP and HTTPS listeners, got {listeners!r}" ) assert listeners["http"].get("protocol") == "HTTP" and listeners["http"].get("port") == 80 assert listeners["https"].get("protocol") == "HTTPS" and listeners["https"].get("port") == 443 assert listeners["http"].get("hostname") == listeners["https"].get("hostname"), ( f"Test setup must use a shared hostname across listeners: {listeners!r}" ) route_name = _tenant_api_names(case)["httproute"] route = wait_for_json( "httproute", route_name, INFRA_NAMESPACE, predicate=lambda obj: _route_accepted_by_gateway(obj, case["gateway_name"]), timeout=180, ) authconfigs_by_rule = _wait_for_stable_route_authconfigs(route) rules = (route.get("spec") or {}).get("rules") or [] expected_rules = {f"rule-{index}" for index in range(1, len(rules) + 1)} assert set(authconfigs_by_rule) == expected_rules > assert all( len(observations) == 1 and observations[0][2] for observations in authconfigs_by_rule.values() ), ( f"Expected exactly one Ready and Available non-deleting Authorino " f"AuthConfig per rule for " f"{INFRA_NAMESPACE}/{route_name}; got {authconfigs_by_rule!r}" ) E AssertionError: Expected exactly one Ready and Available non-deleting Authorino AuthConfig per rule for odh-ai-gateway-infra/maas-api-route-e2e-authcfg-http-05bcf4; got {'rule-2': [('0e196bb792f90df8229c7e34b890c37fa8b72f2ef90f0c69f6b0a1ae41da7f1e', '6a1d7aee-5545-47df-9bad-2c66aa71fda8', True), ('5f7c5ae79f09c10a6acf8edf5e00447aba6ff9c27d5c4e17ed8701dabfc81329', 'e5e2be79-f613-4168-99c0-8c0d2b8fe633', True)], 'rule-5': [('192608f96ddebb161b7a85c99ff15e1a54fe9cd07c4c240269c23e8065a6cf93', '03546f68-797d-49ee-9a87-07147c0132f9', True), ('7e251036b2d29347f9004e05e5f5428e95596072aae9105de110b7c10be81157', '5112ea78-1fd0-4bb0-baba-0e8355e4641d', True)], 'rule-3': [('23c7ef95750d2cbd97a8cb81cf91a8fbe4e91c42b91d5175381d2dbac2287fb0', '589e693c-3c0a-4b8b-bebf-a0239e55be45', True), ('5135bb52761235c991221359f4b8935de395ff2b956ab88c65c7d99ece6d391b', 'bbc26103-ec46-4d8f-8e4c-ddca65a79da8', True)], 'rule-6': [('46d324d6c585aa15198c0d4fa79c5cabfad11e8ba076e2f396a45dba3c342324', 'fc51d959-6761-4122-8fb0-67297ebd0a3b', True), ('5661f627fcb25c67fbbcc2828cffa4d483f79faec5e5a6dd99b1bb89586f8dd1', 'fff9591a-2293-4866-ab58-2516b15345d3', True)], 'rule-1': [('535096dedee6dfd65e8d1fd21c4153e501f4b7c4ee6016faa5117fd375b29097', 'f735d368-dd2b-44c4-9f4b-e425f0cdef61', True), ('83c8e70660c5e22ff710a7d51ef99ae16cb22d56167a60609b548939ddbdd6f5', '3c664f93-8846-4a1f-958f-d505a7d2bee5', True)], 'rule-4': [('757b4eaf7cdce987cf205d91d43022fede732e2e11227cc806a9758674e71a45', 'd6057240-5391-4b71-bd28-c2cd17c75f7f', True), ('bd153a6e20252138d9b173b440575adfa887bd5ae5638c822eb187dab3ae8f6b', '1636e0c5-86a6-4fa9-aa65-5261167bbb39', True)]} E assert False E + where False = all(<generator object TestPerTenantMaaSAPI.test_dual_listeners_keep_one_authconfig_per_route_rule.<locals>.<genexpr> at 0x7f791159dee0>) test/maas-e2e/test/e2e/tests/test_multi_tenant_maas_api.py:381: AssertionError