/workspace/source/test/maas-e2e/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/maas-e2e/test/e2e/tests/test_subscription.py:1580: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestE2ESubscriptionFlow object at 0x7f5f1af7ae10> def test_e2e_with_both_access_and_subscription_gets_200(self): """ API key with both access and subscription should get 200 OK. Uses an existing model reference that aligns with gateway path-based model identity. """ ns = _ns() auth_policy_name = "e2e-test-auth-success" subscription_name = "e2e-test-subscription-success" sa_name = "e2e-sa-success" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Use pre-existing premium model to keep model identity consistent with gateway path. _create_test_auth_policy(auth_policy_name, PREMIUM_MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, PREMIUM_MODEL_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth_policy_name, namespace=ns, timeout=120, require_auth_policies=False) > _wait_for_subscription_inference_ready( subscription_name, PREMIUM_MODEL_REF, namespace=ns, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:1887: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'premium-w3-306aed' model_namespace = 'e2e-models-e2e-worker-w3-306aed', timeout = 180 subscription_name = 'e2e-test-subscription-success' subscription_namespace = 'ai-tenant-e2e-worker-w3-306aed' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-premium-w3-306aed was not created and enforced in e2e-models-e2e-worker-w3-306aed within 180s: generation=2, observedGeneration=2, Enforced={'lastTransitionTime': '2026-10-30T17:28:42Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutErrorself = <test_subscription.TestE2ESubscriptionFlow object at 0x7efeb3c47390> def test_e2e_group_based_access_gets_200(self): """ E2E test: Group-based auth and subscription (success case). Validates that users can access models via group membership in both MaaSAuthPolicy and MaaSSubscription, not just explicit user lists. """ ns = _ns() auth_policy_name = "e2e-test-group-auth" subscription_name = "e2e-test-group-subscription" sa_name = "e2e-sa-group" # Use namespace-specific group that SA will be in test_group = f"system:serviceaccounts:{ns}" try: # Create service account and get OC token for maas-api oc_token = _create_sa_token(sa_name, namespace=ns) # Create auth policy using GROUP (not user) _create_test_auth_policy(auth_policy_name, MODEL_REF, groups=[test_group]) # Create subscription using GROUP (not user) _create_test_subscription(subscription_name, MODEL_REF, groups=[test_group]) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) > _wait_for_subscription_inference_ready( subscription_name, MODEL_REF, model_namespace=MODEL_NAMESPACE, timeout=180, ) test/maas-e2e/test/e2e/tests/test_subscription.py:2216: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/maas-e2e/test/e2e/tests/test_helper.py:1781: in _wait_for_subscription_inference_ready _wait_for_token_rate_limit_policy( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ model_ref = 'simulator-w2-506124' model_namespace = 'e2e-models-e2e-worker-w2-506124', timeout = 180 subscription_name = 'e2e-test-group-subscription' subscription_namespace = 'ai-tenant-e2e-worker-w2-506124' def _wait_for_token_rate_limit_policy( model_ref, model_namespace=MODEL_NAMESPACE, timeout=60, *, subscription_name=None, subscription_namespace=None ): """Wait for the current TokenRateLimitPolicy generation to be enforced. If a subscription is supplied, wait until the policy includes it. This prevents accepting the old policy before the controller updates its spec. Args: model_ref: Name of the model (e.g., "e2e-distinct-simulated") model_namespace: Namespace where the TRLP should be created (default: MODEL_NAMESPACE) timeout: Maximum wait time in seconds (default: 60) subscription_name: Subscription that the policy must include, if supplied subscription_namespace: Subscription namespace (defaults to _ns()) Raises: TimeoutError: If TRLP isn't created and enforced within timeout """ trlp_name = f"maas-trlp-{model_ref}" expected_subscription = ( f"{subscription_namespace or _ns()}/{subscription_name}" if subscription_name else None ) last_snapshot = "not found" deadline = time.time() + timeout log.info(f"Waiting for TokenRateLimitPolicy {trlp_name} in {model_namespace} (timeout: {timeout}s)...") while time.time() < deadline: result = subprocess.run( ["oc", "get", "tokenratelimitpolicy", trlp_name, "-n", model_namespace, "-o", "json"], capture_output=True, text=True, timeout=30, ) if result.returncode == 0: try: trlp = json.loads(result.stdout) metadata = trlp.get("metadata", {}) status = trlp.get("status", {}) generation = metadata.get("generation") observed = status.get("observedGeneration") conditions = status.get("conditions", []) enforced = next((c for c in conditions if c.get("type") == "Enforced"), None) subscriptions = { name.strip() for name in metadata.get("annotations", {}).get( "maas.opendatahub.io/subscriptions", "" ).split(",") } includes_subscription = expected_subscription is None or expected_subscription in subscriptions last_snapshot = ( f"generation={generation}, observedGeneration={observed}, " f"Enforced={enforced}, includes_subscription={includes_subscription}" ) if ( enforced and enforced.get("status") == "True" and generation is not None and observed == generation and includes_subscription ): log.info(f"TokenRateLimitPolicy {trlp_name} generation {generation} is enforced") return log.debug(f"TokenRateLimitPolicy {trlp_name} is not ready: {last_snapshot}") except (json.JSONDecodeError, KeyError) as e: log.debug(f"Failed to parse TRLP status: {e}") elif _is_not_found_error(result.stderr): log.debug(f"TokenRateLimitPolicy {trlp_name} not found yet...") elif _is_transient_kubectl_error(result.stderr): log.debug( f"Transient error while reading TokenRateLimitPolicy {trlp_name}: {result.stderr.strip()}" ) else: raise RuntimeError( f"Failed to get TokenRateLimitPolicy {trlp_name} in namespace '{model_namespace}': " f"{result.stderr.strip()}" ) time.sleep(3) > raise TimeoutError( f"TokenRateLimitPolicy {trlp_name} was not created and enforced in {model_namespace} within {timeout}s: {last_snapshot}" ) E TimeoutError: TokenRateLimitPolicy maas-trlp-simulator-w2-506124 was not created and enforced in e2e-models-e2e-worker-w2-506124 within 180s: generation=4, observedGeneration=4, Enforced={'lastTransitionTime': '2026-10-30T17:28:30Z', 'message': 'TokenRateLimitPolicy waiting for the following components to sync: [Limitador]', 'reason': 'Unknown', 'status': 'False', 'type': 'Enforced'}, includes_subscription=True test/maas-e2e/test/e2e/tests/test_helper.py:1538: TimeoutError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_namespace_discovery.py:190: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/maas-e2e/test/e2e/tests/test_gateway_scoped_authpolicy.py:213: legacy gateway-default-auth is scoped to the shared default gatewayself = <test_multi_tenant_integration.TestMultiTenantIntegration object at 0x7f078b809d10> def test_full_tenant_lifecycle_create_to_delete(self): """7.1: Full tenant lifecycle from create through policy/subscription reconcile to delete.""" case = new_discovery_case() role_name = f"aitenant-{case['tenant_label_name']}-tenant-admin" try: bootstrap_aitenant_tenant(case) tenant = wait_for_json(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) tenant_labels = tenant["metadata"].get("labels") or {} tenant_annotations = tenant["metadata"].get("annotations") or {} assert tenant_labels[LABEL_MANAGED_BY_AITENANT] == "true" assert tenant_labels[LABEL_TENANT_NAME] == case["tenant_label_name"] assert tenant_labels[LABEL_TENANT_NAMESPACE] == case["tenant_ns"] assert tenant_annotations[ANNOTATION_AITENANT_NAME] == case["tenant_label_name"] assert tenant_annotations[ANNOTATION_AITENANT_NAMESPACE] == AITENANT_NAMESPACE aitenant = wait_for_json(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout=180) assert aitenant["status"]["gatewayRef"]["name"] == case["gateway_name"] assert get_json_or_none("role", role_name, case["tenant_ns"]) is not None model_name = f"e2e-lifecycle-model-{case['suffix']}" provision_tenant_model(model_name, case["tenant_ns"], case["gateway_name"]) apply_maas_auth_policy( case["policy_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) apply_maas_subscription( case["subscription_name"], case["tenant_ns"], model_ref=model_name, model_namespace=case["tenant_ns"], ) wait_for_status_phase("maasauthpolicy", case["policy_name"], case["tenant_ns"], expected_phase="Active") wait_for_status_phase( "maassubscription", case["subscription_name"], case["tenant_ns"], expected_phase="Active", ) wait_for_aitenant_cleanup_resources(case) # Bind the key to the tenant's subscription, then verify it remains # invalid after the tenant and its subscription are recreated. route = wait_for_route_admitted(f"{case['gateway_name']}-route") tenant_api_url = f"https://{route['spec']['host']}/maas-api" wait_for_deployment_available(f"maas-api-{case['tenant_label_name']}", timeout=180) key_response = create_api_key_at( tenant_api_url, _get_cluster_token(), f"e2e-aitenant-delete-{case['suffix']}", subscription=case["subscription_name"], ) assert key_response.status_code in (200, 201), ( f"failed to create tenant-scoped API key: {response_summary(key_response)}" ) old_api_key = key_response.json().get("key") assert old_api_key, "tenant-scoped API key response did not include a key" validation = validate_api_key_at( tenant_internal_url(case["tenant_label_name"]), old_api_key, ) assert validation.status_code == 200 and validation.json().get("valid") is True, ( f"new tenant API key did not validate: status={validation.status_code} " f"body={validation.text}" ) user_objects = apply_unrelated_tenant_objects(case) delete_best_effort(AITENANT_KIND, case["tenant_label_name"], AITENANT_NAMESPACE, timeout="180s") wait_for_not_found(TENANT_CONFIG_KIND, TENANT_CR_NAME, case["tenant_ns"], timeout=180) wait_for_not_found("maasauthpolicy", case["policy_name"], case["tenant_ns"], timeout=180) wait_for_not_found("maassubscription", case["subscription_name"], case["tenant_ns"], timeout=180) wait_for_not_found("role", role_name, case["tenant_ns"], timeout=180) namespace = wait_for_json( "namespace", case["tenant_ns"], predicate=lambda obj: LABEL_MANAGED_BY_AITENANT not in ((obj.get("metadata") or {}).get("labels") or {}), timeout=180, ) namespace_metadata = namespace.get("metadata") or {} namespace_labels = namespace_metadata.get("labels") or {} namespace_annotations = namespace_metadata.get("annotations") or {} for key in ( "app.kubernetes.io/managed-by", "app.kubernetes.io/part-of", LABEL_AI_GATEWAY_TENANT, LABEL_MANAGED_BY_AITENANT, LABEL_TENANT_NAME, LABEL_TENANT_NAMESPACE, "opendatahub.io/generated-namespace", ): assert key not in namespace_labels for key in ( ANNOTATION_AITENANT_NAME, ANNOTATION_AITENANT_NAMESPACE, "maas.opendatahub.io/created-by-aitenant", ): assert key not in namespace_annotations assert get_json_or_none("secret", user_objects["secret"], case["tenant_ns"]) is not None assert get_json_or_none("rolebinding", user_objects["rolebinding"], case["tenant_ns"]) is not None wait_for_aitenant_cleanup_resources_deleted(case) # Recreate the same tenant name and verify the old key cannot be # used against the newly provisioned maas-api instance. bootstrap_aitenant_tenant(case) wait_for_aitenant_cleanup_resources(case) wait_for_deployment_available(f"maas-api-{case['tenant_label_name']}", timeout=180) validation = validate_api_key_at( tenant_internal_url(case["tenant_label_name"]), old_api_key, ) > assert validation.status_code == 200 and validation.json().get("valid") is False, ( "API key from the deleted tenant remained valid after recreating " f"the tenant with the same name: status={validation.status_code} " f"body={validation.text}" ) E AssertionError: API key from the deleted tenant remained valid after recreating the tenant with the same name: status=0 body=Command '['kubectl', 'exec', '-i', 'test-curl-8860-25f504', '-n', 'opendatahub', '--', 'sh', '-c', 'cat > /tmp/ca-bundle.crt']' timed out after 15 seconds E assert (0 == 200) E + where 0 = <multitenancy_helpers._InternalResponse object at 0x7f078b6b70d0>.status_code test/maas-e2e/test/e2e/tests/test_multi_tenant_integration.py:196: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners.tenant_service_urls = {'tenant_a': {'name': 'e2e-shared-a-w2-a17354', 'namespace': 'ai-tenant-e2e-shared-a-w2-a17354', 'service_url': 'https...as-api-e2e-shared-b-w2-016f1d.odh-ai-gateway-infra.svc.cluster.local:8443', 'aitenant_name': 'e2e-shared-b-w2-016f1d'}} def test_tenant_discovery_unauthorized_access(tenant_service_urls): """ Verify completely unauthorized access is rejected. Without any token or with an invalid token, all tenant endpoints should return 401. """ for tenant_key in ["tenant_a", "tenant_b"]: tenant = tenant_service_urls[tenant_key] url = f"{tenant['service_url']}/v1/tenants" # Test 1: No auth header status_code, _ = _kubectl_curl(url) > assert status_code == 401, \ f"{tenant['name']} should reject no-auth request with 401, got {status_code}" E AssertionError: e2e-shared-b-w2-016f1d should reject no-auth request with 401, got 0 E assert 0 == 401 test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:206: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/maas-e2e/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_Bself = <test_tenant_subscription_isolation.TestTenantSubscriptionIsolation object at 0x7f0a18658310> tenant_subscriptions = {'shared': 'e2e-shared-sub-d9a25f', 'tenant_a_only': 'e2e-a-only-d9a25f', 'tenant_b_only': 'e2e-b-only-d9a25f', 'tenan...hared-a-w0-c31c73', 'tenant_label_name': 'e2e-shared-a-w0-c31c73', 'gateway_name': 'e2e-shared-a-w0-c31c73', ...}, ...} def test_subscription_selection_per_tenant(self, tenant_subscriptions): """4.2: Same-named subscriptions resolve to the namespace behind each tenant endpoint.""" shared = tenant_subscriptions["shared"] tenant_a = tenant_subscriptions["tenant_a"] tenant_b = tenant_subscriptions["tenant_b"] key_a = _create_key_for_subscription(tenant_a, shared) key_b = _create_key_for_subscription(tenant_b, shared) requested_model_a = f"{tenant_a['model_namespace']}/{tenant_a['model_name']}" response_a = select_subscription_at( tenant_internal_url(tenant_a["name"]), key_a, "e2e-sub-user", ["system:authenticated"], requested_subscription=shared, requested_model=requested_model_a, ) assert response_a.status_code == 200 data_a = response_a.json() assert data_a.get("error") is None, redact_sensitive(data_a) assert data_a.get("name") == shared assert data_a.get("namespace") == tenant_a["namespace"] requested_model_b = f"{tenant_b['model_namespace']}/{tenant_b['model_name']}" response_b = select_subscription_at( tenant_internal_url(tenant_b["name"]), key_b, "e2e-sub-user", ["system:authenticated"], requested_subscription=shared, requested_model=requested_model_b, ) > assert response_b.status_code == 200 E assert 0 == 200 E + where 0 = <multitenancy_helpers._InternalResponse object at 0x7f0a1868a110>.status_code test/maas-e2e/test/e2e/tests/test_tenant_subscription_isolation.py:219: AssertionError/workspace/source/test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)/workspace/source/test/maas-e2e/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable