/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_subscription.TestE2ESubscriptionFlow object at 0x7fef51f7eed0> def test_e2e_multiple_subscriptions_separate_keys_gets_200(self): """ User with two subscriptions for the same model: mint one API key per subscription; each key succeeds on inference without x-maas-subscription. """ ns = _ns() auth_policy_name = "e2e-test-auth-multi-sub-valid" subscription_1 = "e2e-test-subscription-free" subscription_2 = "e2e-test-subscription-premium" sa_name = "e2e-sa-multi-sub-valid" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources with 2 subscriptions for the same user _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_1, MODEL_REF, users=[sa_user], token_limit=100) _create_test_subscription(subscription_2, MODEL_REF, users=[sa_user], token_limit=1000) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_1, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2089: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w4-9c901b' model_namespace = 'e2e-models-e2e-worker-w4-9c901b', timeout = 180 subscription_name = 'e2e-test-subscription-free' subscription_namespace = 'ai-tenant-e2e-worker-w4-9c901b' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w4-9c901b was not created and enforced in e2e-models-e2e-worker-w4-9c901b within 180s: generation=5, observedGeneration=5, Enforced={'lastTransitionTime': '2026-10-30T20:17:57Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7fd2db189790> def test_e2e_with_both_access_and_subscription_gets_200(self): """ API key with both access and subscription should get 200 OK. Uses an existing model reference that aligns with gateway path-based model identity. """ ns = _ns() auth_policy_name = "e2e-test-auth-success" subscription_name = "e2e-test-subscription-success" sa_name = "e2e-sa-success" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Use pre-existing premium model to keep model identity consistent with gateway path. _create_test_auth_policy(auth_policy_name, PREMIUM_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, PREMIUM_MODEL_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth_policy_name, namespace=ns, timeout=120, require_auth_policies=False) > _wait_for_subscription_inference_ready( subscription_name, PREMIUM_MODEL_REF, namespace=ns, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1887: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'premium-w0-c646da' model_namespace = 'e2e-models-e2e-worker-w0-c646da', timeout = 180 subscription_name = 'e2e-test-subscription-success' subscription_namespace = 'ai-tenant-e2e-worker-w0-c646da' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-premium-w0-c646da was not created and enforced in e2e-models-e2e-worker-w0-c646da within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T20:18:33Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7f4497f82110> def test_e2e_group_based_access_gets_200(self): """ E2E test: Group-based auth and subscription (success case). Validates that users can access models via group membership in both MaaSAuthPolicy and MaaSSubscription, not just explicit user lists. """ ns = _ns() auth_policy_name = "e2e-test-group-auth" subscription_name = "e2e-test-group-subscription" sa_name = "e2e-sa-group" # Use namespace-specific group that SA will be in test_group = f"system:serviceaccounts:{ns}" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) # Create auth policy using GROUP (not user) _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group]) # Create subscription using GROUP (not user) _create_test_subscription(subscription_name, MODEL_REF, groups=[test_group]) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_name, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2216: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w1-5197ec' model_namespace = 'e2e-models-e2e-worker-w1-5197ec', timeout = 180 subscription_name = 'e2e-test-group-subscription' subscription_namespace = 'ai-tenant-e2e-worker-w1-5197ec' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w1-5197ec was not created and enforced in e2e-models-e2e-worker-w1-5197ec within 180s: generation=4, observedGeneration=4, Enforced={'lastTransitionTime': '2026-10-30T20:17:45Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1580: gateway-only mode: per-model AuthPolicy is not createdmaas_api_base_url = 'https://maas.apps.3d1a0c34cf950dbd372c.prod.konflux-ocp-ci.dev/maas-api' def test_tokens_endpoint_replaced_by_api_keys(maas_api_base_url: str): """ Verify /v1/tokens endpoint has been replaced by the API key system. The endpoint may return 401 (auth required), 404 (removed), or 405 (method not allowed). Any of these indicates the old token minting is no longer available. """ url = f"{maas_api_base_url}/v1/tokens" r = requests.post(url, json={"expiration": "1m"}, timeout=20, verify=TLS_VERIFY) msg = f"[token] POST {url} (no auth) -> {r.status_code}" log.info(msg) print(msg) # 401/404/405 all indicate the old token minting endpoint is gone/protected > assert r.status_code in (401, 404, 405), f"Expected 401/404/405 (endpoint replaced), got {r.status_code}: {r.text[:400]}" E AssertionError: Expected 401/404/405 (endpoint replaced), got 500: Internal Server Error. E E assert 500 in (401, 404, 405) E + where 500 = <Response [500]>.status_code test/maas-e2e/test/e2e/tests/test_smoke.py:50: AssertionErrormaas_api_base_url = 'https://maas.apps.3d1a0c34cf950dbd372c.prod.konflux-ocp-ci.dev/maas-api' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Ik9Va1JFSmZSUXBtYjFJX1lOSmg3eDdpTE96YzY3Y293WEItaEFhQ3FuNWMifQ.e...1fLuSLF4krv3X7YHMr3-YStPhNfTw8yLS1wTVARyjkUDN2zWtrpabPElrK5WRz4hsSW4sZIPaW-IQXB6Q', 'Content-Type': 'application/json'} @pytest.fixture(scope="session") def model_catalog(maas_api_base_url: str, headers: dict): r = requests.get(f"{maas_api_base_url}/v1/models", headers=headers, timeout=45, verify=TLS_VERIFY) > r.raise_for_status() test/maas-e2e/test/e2e/tests/conftest.py:276: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Response [500]> def raise_for_status(self) -> None: """Raises :class:`HTTPError`, if one occurred.""" http_error_msg = "" if isinstance(self.reason, bytes): # We attempt to decode utf-8 first because some servers # choose to localize their reason strings. If the string # isn't utf-8, we fall back to iso-8859-1 for all other # encodings. (See PR #3538) try: reason = self.reason.decode("utf-8") except UnicodeDecodeError: reason = self.reason.decode("iso-8859-1") else: reason = self.reason if 400 <= self.status_code < 500: http_error_msg = ( f"{self.status_code} Client Error: {reason} for url: {self.url}" ) elif 500 <= self.status_code < 600: http_error_msg = ( f"{self.status_code} Server Error: {reason} for url: {self.url}" ) if http_error_msg: > raise HTTPError(http_error_msg, response=self) E requests.exceptions.HTTPError: 500 Server Error: Internal Server Error for url: https://maas.apps.3d1a0c34cf950dbd372c.prod.konflux-ocp-ci.dev/maas-api/v1/models test/e2e/.venv/lib64/python3.11/site-packages/requests/models.py:1167: HTTPError