/workspace/source/test/e2e/tests/test_api_keys.py:930: CronJob maas-api-key-cleanup not found in opendatahub: Error from server (NotFound): cronjobs.batch "maas-api-key-cleanup" not found/workspace/source/test/e2e/tests/test_api_keys.py:977: NetworkPolicy maas-api-cleanup-restrict not found in opendatahub: Error from server (NotFound): networkpolicies.networking.k8s.io "maas-api-cleanup-restrict" not found/workspace/source/test/e2e/tests/test_api_keys.py:1096: Cannot find maas-api pod in opendatahub: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}self = <test_api_keys.TestAPIKeySubscriptionFilter object at 0x7fd3adadb9d0> api_keys_base_url = 'https://maas.apps.0a3c0600-319c-4f25-a3dd-4d87622c2546.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjBtTE1qckMxVFpMWGx2SjBfakVVRUNGVDI1VUlWNVpsQ3kzbUw0UTlDT0EifQ.e...P0Bgp_4HBRDKtRtPk_s-Bo_MV6UTpG1WVQdJP4a6DK3CFLswSvQnKfXtXc7-KlHLM4TVdeWxEmfbziT8g', 'Content-Type': 'application/json'} def test_search_without_subscription_returns_all(self, api_keys_base_url: str, headers: dict): """Search without subscription filter returns keys across all subscriptions.""" key_ids = [] try: # Create keys with explicit subscription binding for i in range(2): r = requests.post( api_keys_base_url, headers=headers, json={"name": f"e2e-nofilter-{i}", "subscription": SIMULATOR_SUBSCRIPTION}, timeout=TIMEOUT, verify=TLS_VERIFY, ) > assert r.status_code in (200, 201), f"Failed to create key: {r.text}" E AssertionError: Failed to create key: E assert 403 in (200, 201) E + where 403 = <Response [403]>.status_code test/e2e/tests/test_api_keys.py:1591: AssertionError/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestHeaderSpoofing object at 0x7fd3ad610880> def test_injected_identity_headers_ignored(self): """Client injects X-MaaS-Username/Group/Key-Id — platform ignores them. Validates that Authorino strips attacker-controlled identity headers. The request should succeed (200) using the real key-derived identity, proving the spoofed headers had no effect on authorization. """ > api_key = _create_api_key(_get_cluster_token(), subscription=SIMULATOR_SUBSCRIPTION) test/e2e/tests/test_negative_security.py:83: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjBtTE1qckMxVFpMWGx2SjBfakVVRUNGVDI1VUlWNVpsQ3kzbUw0UTlDT0EifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm..._db27Ae_aPZk9SlFCBGVgRd7TeNNlVz446U5jP0Bgp_4HBRDKtRtPk_s-Bo_MV6UTpG1WVQdJP4a6DK3CFLswSvQnKfXtXc7-KlHLM4TVdeWxEmfbziT8g' name = None, subscription = 'simulator-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 403 test/e2e/tests/test_helper.py:247: RuntimeErrorself = <test_negative_security.TestHeaderSpoofing object at 0x7fd3ad6103a0> def test_duplicate_subscription_headers_ignored(self): """Client sends multiple X-MaaS-Subscription headers — API key binding wins. For API key requests, the subscription is fixed at mint time. Duplicate or conflicting X-MaaS-Subscription headers must not override the key-derived subscription. """ > api_key = _create_api_key(_get_cluster_token(), subscription=SIMULATOR_SUBSCRIPTION) test/e2e/tests/test_negative_security.py:108: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjBtTE1qckMxVFpMWGx2SjBfakVVRUNGVDI1VUlWNVpsQ3kzbUw0UTlDT0EifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm..._db27Ae_aPZk9SlFCBGVgRd7TeNNlVz446U5jP0Bgp_4HBRDKtRtPk_s-Bo_MV6UTpG1WVQdJP4a6DK3CFLswSvQnKfXtXc7-KlHLM4TVdeWxEmfbziT8g' name = None, subscription = 'simulator-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 403 test/e2e/tests/test_helper.py:247: RuntimeError/workspace/source/test/e2e/tests/test_negative_security.py:188: Could not create short-lived key: 403self = <test_negative_security.TestCrossModelAccess object at 0x7fd3ad610550> def test_key_cannot_access_model_outside_subscription(self): """Key for model A cannot infer on model B outside its subscription. Uses the pre-deployed unconfigured model (a model with no subscription granting access to it) to test cross-model access denial. """ > api_key = _create_api_key(_get_cluster_token(), subscription=SIMULATOR_SUBSCRIPTION) test/e2e/tests/test_negative_security.py:222: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ oc_token = 'eyJhbGciOiJSUzI1NiIsImtpZCI6IjBtTE1qckMxVFpMWGx2SjBfakVVRUNGVDI1VUlWNVpsQ3kzbUw0UTlDT0EifQ.eyJhdWQiOlsiaHR0cHM6Ly9wcm..._db27Ae_aPZk9SlFCBGVgRd7TeNNlVz446U5jP0Bgp_4HBRDKtRtPk_s-Bo_MV6UTpG1WVQdJP4a6DK3CFLswSvQnKfXtXc7-KlHLM4TVdeWxEmfbziT8g' name = None, subscription = 'simulator-subscription' def _create_api_key(oc_token: str, name: str = None, subscription: str = None) -> str: """Create an API key using the MaaS API and return the plaintext key. Args: oc_token: OC token for authentication with maas-api name: Optional name for the key (auto-generated if not provided) subscription: Optional MaaSSubscription name to bind (highest-priority auto-bind if omitted) Returns: The plaintext API key (sk-oai-xxx format) """ r = _create_api_key_raw(oc_token, name, subscription) if r.status_code not in (200, 201): > raise RuntimeError(f"Failed to create API key: {r.status_code} {r.text}") E RuntimeError: Failed to create API key: 403 test/e2e/tests/test_helper.py:247: RuntimeError/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsgateway_host = 'maas.apps.0a3c0600-319c-4f25-a3dd-4d87622c2546.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-1f446e' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-1f446e in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionError