/workspace/source/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/e2e/tests/test_external_model_routing.py:564: provider withdrawal E2E for #106 is gated; set TENANT_EXTERNAL_MODEL_LAST_PROVIDER_WITHDRAWAL_E2E=true to enable (with it set, the #106 contract is expected to hold and a violation fails red)/workspace/source/test/e2e/tests/test_external_model_routing.py:481: provider withdrawal E2E for #108 is gated; set TENANT_EXTERNAL_MODEL_PARTIAL_WITHDRAWAL_E2E=true to enable (with it set, the #108 contract is expected to hold and a violation fails red)self = <test_models_endpoint.TestModelsEndpoint object at 0x7efc65048110> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403. The gateway may reject the revoked key before the request reaches maas-api, in which case the response body is empty. """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_subscription_discovery_ready(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_for_cr_absent("maassubscription", subscription_name, namespace=ns) # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist > assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for API key with deleted subscription, got 500: Internal Server Error. E E assert 500 == 403 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_models_endpoint.py:1655: AssertionError/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_networkpolicy.py:119: AIGC praxis NetworkPolicy does not include OTLP 4317 egress (MaaS Go IPP bundle only)/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicable/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)self = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f05952f9c50> def test_degraded_healthy_model_allows_inference(self): """ Test: Inference to healthy model in Degraded subscription succeeds. Setup: 1. Create subscription with 1 valid + 1 missing model 2. Controller sets phase=Degraded, modelRefStatuses shows mixed health Verify: - Subscription is Degraded with one ready=true, one ready=false - Inference to the valid model succeeds (200) """ ns = _ns() subscription_name = "e2e-degraded-healthy-inf" auth_name = "e2e-degraded-healthy-inf-auth" sa_name = "e2e-degraded-healthy-inf-sa" missing_model = "nonexistent-model-inf" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model only _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid + missing → auto-Degraded _create_test_subscription( subscription_name, [MODEL_REF, missing_model], users=[sa_user] ) cr = _wait_for_maas_subscription_phase(subscription_name, "Degraded") # Verify Degraded with mixed health status = cr.get("status", {}) phase = status.get("phase") model_statuses = status.get("modelRefStatuses", []) log.info(f"Phase: {phase}, modelRefStatuses: {model_statuses}") assert phase == "Degraded", f"Expected Degraded, got {phase}" assert len(model_statuses) == 2, f"Expected 2 statuses, got {len(model_statuses)}" # Find our valid model status valid_status = next( (s for s in model_statuses if s.get("name") == MODEL_REF), None ) assert valid_status is not None, f"Missing status for {MODEL_REF}" assert valid_status.get("ready") is True, \ f"Expected {MODEL_REF} ready=true, got {valid_status}" log.info(f"✅ Subscription Degraded with {MODEL_REF} healthy") # Create API key # oc_token already set from _create_sa_token above api_key = _create_api_key( oc_token, name="degraded-healthy", subscription=subscription_name ) # Inference to healthy model should work log.info(f"Testing inference to healthy {MODEL_REF}...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) > assert r.status_code == 200, \ f"Expected 200 for healthy model in Degraded subscription, got {r.status_code}: {r.text[:500]}" E AssertionError: Expected 200 for healthy model in Degraded subscription, got 500: Internal Server Error. E E assert 500 == 200 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_subscription.py:3060: AssertionError/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:223: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_B/workspace/source/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:199: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:268: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:307: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:341: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:396: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:434: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:227: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_crossns.py:251: cross-namespace provider E2E is gated until AGC ships cross-namespace provider support; set TENANT_EXTERNAL_MODEL_CROSSNS_E2E=true to enableself = <test_tenant_external_model_conflicts.TestForeignBrokenModelIsolation object at 0x7f95994aa010> conflicts_env = {'tenant_a': {'suffix': 'da409b', 'tenant_ns': 'ai-tenant-e2e-extmod-a-da409b', 'tenant_label_name': 'e2e-extmod-a-da4...tmod-cfl-2-94165a', 'resources': <tenant_external_model_helpers.TenantExternalModelResources object at 0x7f95991ff110>} broken_pair = {'healthy': {'name': 'e2e-cfl-ok-3c9526', 'namespace': 'e2e-extmod-cfl-1-4c3c6d', 'client_name': 'e2e-cfl-ok-3c9526', ...i_key': 'sk-oai-3PixGNSmnCUdm9KX_6fY87jvAGGmI8827JzdsOrCkiVhgB8rkKynBs7mIfpd'}, 'broken_name': 'e2e-cfl-broken-36335f'} def test_broken_foreign_model_does_not_break_tenant_models(self, conflicts_env, broken_pair): """The broken foreign attachment reports a failure while the tenant's own healthy model on the same gateway keeps serving.""" env = conflicts_env tenant_a = env["tenant_a"] broken_ns = env["user_ns_2"] wait_for_gateway_attachment_not_ready( broken_pair["broken_name"], broken_ns, tenant_a["gateway_name"] ) healthy = broken_pair["healthy"] response = infer_external_chat( tenant_a["gateway_url"], healthy["namespace"], healthy["client_name"], broken_pair["access"]["api_key"], path_tag=healthy["path_tag"], ) > assert_served_by_provider(response, healthy["path_tag"], credential=healthy["credential"]) test/e2e/tests/test_tenant_external_model_conflicts.py:457: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ response = <Response [429]>, path_tag = 'e2e-cfl-ok-3c9526-a4220c' credential = 'sk-extmod-a4220c' def assert_served_by_provider(response: requests.Response, path_tag: str, *, credential: str | None = None) -> None: """Assert a request reached the expected provider (and credential). The echo endpoint reflects the outgoing request URL and headers in the response body: the path tag confirms the path the client requested is the path the provider received (the gateway rewrites only the ``/{namespace}/{modelName}`` prefix), and the credential marker identifies the injected Secret — i.e. which provider binding served. """ > assert response.status_code == 200, ( ^^^^^^^^^^^^^^^^^^^^^^^^^^^ f"expected 200 from external model inference, got {response.status_code}: " f"{redact_sensitive(response.text)}" ) E AssertionError: expected 200 from external model inference, got 429: Too Many Requests test/e2e/tests/external_model_helpers.py:495: AssertionError