/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestHeaderSpoofing object at 0x7fcad9ddf2e0> def test_duplicate_subscription_headers_ignored(self): """Client sends multiple X-MaaS-Subscription headers — API key binding wins. For API key requests, the subscription is fixed at mint time. Duplicate or conflicting X-MaaS-Subscription headers must not override the key-derived subscription. """ _wait_for_gateway_auth_enforced() api_key = _create_api_key(_get_cluster_token(), subscription=SIMULATOR_SUBSCRIPTION) # Warm up: confirm the API key works with a normal request before # testing duplicate headers. Under parallel load, Rego policy # propagation can take longer than the 30s retry window below. _poll_status(api_key, 200, timeout=60) # Use http.client to send genuinely duplicate X-MaaS-Subscription headers. # The requests library uses a dict for headers, so it cannot send two # headers with the same name — the second value overwrites the first. path = f"{MODEL_PATH}/v1/completions" body = json.dumps({"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 3}) # Two separate X-MaaS-Subscription header lines headers = [ ("Authorization", f"Bearer {api_key}"), ("Content-Type", "application/json"), ("X-MaaS-Subscription", SIMULATOR_SUBSCRIPTION), ("X-MaaS-Subscription", "nonexistent-fake-sub"), ] # As above, keep the raw duplicate headers on every retry while waiting # for shared gateway authorization state to settle. deadline = time.time() + 30 while True: gateway = _gateway_url() parsed = urlparse(gateway) if parsed.scheme == "https": ctx = ssl.create_default_context() if not TLS_VERIFY: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE conn = http.client.HTTPSConnection( parsed.hostname, parsed.port or 443, timeout=TIMEOUT, context=ctx, ) else: conn = http.client.HTTPConnection( parsed.hostname, parsed.port or 80, timeout=TIMEOUT, ) try: conn.putrequest("POST", path) for key, value in headers: conn.putheader(key, value) conn.putheader("Content-Length", str(len(body))) conn.endheaders(body.encode()) resp = conn.getresponse() status = resp.status resp_body = resp.read().decode(errors="replace") finally: conn.close() if status == 200 or time.time() >= deadline: break time.sleep(2) # API key binding wins — request succeeds with key-derived subscription. log.info("Duplicate X-MaaS-Subscription headers -> %s", status) > assert status == 200, ( f"Expected 200 (API key subscription binding wins over duplicate headers), " f"got {status}: {resp_body[:500]}" ) E AssertionError: Expected 200 (API key subscription binding wins over duplicate headers), got 403: E assert 403 == 200 test/e2e/tests/test_negative_security.py:335: AssertionError/workspace/source/test/e2e/tests/test_tenant_discovery.py:79: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery.py:155: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_external_oidc.py:252: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:263: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:277: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:290: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:303: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:335: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:379: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:395: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:405: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:428: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:436: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:441: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:454: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:501: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:537: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:602: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:633: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:659: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:690: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:750: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:786: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:824: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:872: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:961: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1013: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1049: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1103: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_external_oidc.py:1142: EXTERNAL_OIDC not enabled/workspace/source/test/e2e/tests/test_subscription.py:1069: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription_list_endpoints.TestListSubscriptions object at 0x7fd02496a3a0> def test_model_ref_display_name_and_description_enriched(self): """Model refs include display_name and description from MaaSModelRef annotations.""" sa_name = "e2e-enrichment-sa" sa_ns = "default" maas_ns = _ns() model_ref_name = "e2e-enrichment-model-ref" model_ns = MODEL_NAMESPACE subscription_name = "e2e-enrichment-sub" auth_policy_name = "e2e-enrichment-auth" expected_display_name = "E2E Enrichment Test Model" expected_description = "Model created by e2e test to verify display_name/description enrichment" try: sa_token = _create_sa_token(sa_name, namespace=sa_ns) sa_user = _sa_to_user(sa_name, namespace=sa_ns) # Create a MaaSModelRef with display-name and description annotations. # Points to the existing e2e-distinct-simulated LLMIS so the controller # can reconcile the subscription to Active. _apply_cr({ "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "MaaSModelRef", "metadata": { "name": model_ref_name, "namespace": model_ns, "annotations": { "openshift.io/display-name": expected_display_name, "openshift.io/description": expected_description, }, }, "spec": { "modelRef": { "kind": "LLMInferenceService", "name": "e2e-distinct-simulated", } }, }) _create_test_subscription( subscription_name, model_ref_name, users=[sa_user], ) _create_test_auth_policy( auth_policy_name, model_ref_name, users=[sa_user], ) api_key = _create_api_key(sa_token, name=f"{sa_name}-key") _wait_for_maas_subscription_phase(subscription_name, namespace=maas_ns) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False, namespace=maas_ns) url = f"{_maas_api_url()}/v1/subscriptions" r = requests.get( url, headers={"Authorization": f"Bearer {api_key}"}, timeout=TIMEOUT, verify=TLS_VERIFY, ) > assert r.status_code == 200, f"Expected 200, got {r.status_code}: {r.text}" E AssertionError: Expected 200, got 500: {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"001"} E assert 500 == 200 E + where 500 = <Response [500]>.status_code test/e2e/tests/test_subscription_list_endpoints.py:264: AssertionErrorself = <test_models_endpoint.TestModelsEndpoint object at 0x7f7afd5edbb0> def test_api_key_with_deleted_subscription_403(self): """ Test: API key bound to a subscription that was deleted after key creation. This tests an edge case where an API key was minted with a subscription, but that subscription is later deleted. The gateway injects X-MaaS-Subscription from the key, but the subscription no longer exists. Expected: HTTP 403 with error type: permission_error """ ns = _ns() auth_policy_name = "e2e-api-key-deleted-sub-auth" subscription_name = "e2e-api-key-deleted-sub" sa_name = "e2e-api-key-deleted-sub-sa" api_key = None try: # Create service account and token oc_token = _create_sa_token(sa_name, namespace=ns) sa_user = _sa_to_user(sa_name, namespace=ns) # Create test resources _create_test_auth_policy(auth_policy_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Wait for subscription to reconcile before creating API key _wait_for_maas_subscription_phase(subscription_name, namespace=ns) # Create API key bound to subscription api_key = _create_api_key(oc_token, name=f"{sa_name}-key", subscription=subscription_name) _wait_for_maas_auth_policy_phase(auth_policy_name, require_enforced=False) # Delete the subscription (simulating deletion after key creation) log.info(f"Deleting subscription {subscription_name} after API key creation") _delete_cr("maassubscription", subscription_name, namespace=ns) _wait_for_cr_absent("maassubscription", subscription_name, namespace=ns) # Query with API key (gateway injects deleted subscription name) log.info("Querying /v1/models with API key bound to deleted subscription") r = _request_with_gateway_retry( requests.get, f"{_maas_api_url()}/v1/models", headers={ "Authorization": f"Bearer {api_key}", }, ) # Should return 403 because subscription doesn't exist > assert r.status_code == 403, \ f"Expected 403 for API key with deleted subscription, got {r.status_code}: {r.text}" E AssertionError: Expected 403 for API key with deleted subscription, got 200: {"data":[],"object":"list"} E assert 200 == 403 E + where 200 = <Response [200]>.status_code test/e2e/tests/test_models_endpoint.py:1586: AssertionError/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:189: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:355: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:93: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:141: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_discovery_isolation.py:228: Skipping when Gateway uses ClusterIP + OpenShift Route (unsupported configuration). This mixes incompatible routing paradigms. Gateway has no external hostname in spec.listeners, so /v1/tenants returns an error. Supported configuration: LoadBalancer service with hostname in spec.listeners./workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:215: Per-tenant OIDC tokens unavailable — set OIDC_TOKEN_URL (tenant-a) and OIDC_TOKEN_URL_TENANT_B (tenant-b), or OIDC_TOKEN_TENANT_A / OIDC_TOKEN_TENANT_Bself = <test_tenant_rate_limit_isolation.TestTenantRateLimitIsolation object at 0x7f4d16577e50> tenant_rate_limit_setup = {'key_a': 'sk-oai-4ar2GsQWo9D09QRf_E5fMNmWEQaoXC3GJT2dO73ZBFVCMpx3E1Bj3amWII6u', 'key_b': 'sk-oai-1Q0CiolNrenxhpaRZ_YJ...VAS3YjGwu9ladf0ImtFkjCYXi4jn63U', 'policy': 'e2e-rate-iso-auth-c609f3', 'subscription_a': 'e2e-rate-iso-a-c609f3', ...} def test_rate_limit_enforced_per_tenant(self, tenant_rate_limit_setup): """5.1: Tenant A's low quota is enforced on Tenant A traffic.""" tenant_a = tenant_rate_limit_setup["tenant_a"] successes, response = _exhaust_until_429( tenant_a["base_url"], tenant_rate_limit_setup["key_a"], tenant_a["model_path"], tenant_a["backend_model_name"], ) > assert successes > 0, f"Tenant A hit 429 before any successful inference: {response_summary(response)}" E AssertionError: Tenant A hit 429 before any successful inference: status=429 body=Too Many Requests E E assert 0 > 0 test/e2e/tests/test_tenant_rate_limit_isolation.py:209: AssertionError