/workspace/source/test/e2e/tests/test_api_keys.py:1083: Cannot exec into maas-api pod to trigger cleanup (neither curl nor wget available): executable file `wget` not found in $PATH: No such file or directory command terminated with exit code 1self = <test_external_oidc.TestOIDCModelAccess object at 0x7f3a570a7580> maas_api_base_url = 'https://maas.apps.04faa770-22fb-475a-b4e9-f5a176fc4161.prod.konfluxeaas.com/maas-api' def test_minted_api_key_can_list_models_and_infer(self, maas_api_base_url: str): """Complete happy path: OIDC token → API key → model list → inference.""" token = _request_oidc_token() > api_key = _create_oidc_api_key(maas_api_base_url, token)["key"] test/e2e/tests/test_external_oidc.py:362: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.04faa770-22fb-475a-b4e9-f5a176fc4161.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJFcnBNRklKVjJtU2RoYVNoenFzdzlwdWF5YlRFWXpCbzNFRzl1cVpzS3pRIn0.eyJle...dV4Bp0rzcWMB2dckbaZa0Ma9BeUty2SXOzNh9sNYq97XqhvEeACWfQ6mwU8GxGiU8q4urSvp37lLqRjL2cDCyvVJ1Ten5iXu7G8EvtLYdyXEoeSBH-Lmyg' name = None, subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token. Retries on empty 403 (gateway propagation delay) and 401 (Authorino may still be loading the OIDC JWKS keys from Keycloak). Args: maas_api_base_url: MaaS API base URL oidc_token: OIDC access token name: Optional key name (auto-generated if omitted) subscription: Optional subscription to bind at mint time Returns: Full response body dict (includes 'key', 'id', 'subscription', etc.) """ body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription retries, delay = 6, 5 for attempt in range(1, retries + 1): response = requests.post( f"{maas_api_base_url}/v1/api-keys", headers={"Authorization": f"Bearer {oidc_token}", "Content-Type": "application/json"}, json=body, timeout=30, verify=TLS_VERIFY, ) # Empty 403: gateway hasn't loaded AuthPolicy yet # 401: Authorino may still be fetching OIDC JWKS keys # 500 AUTH_FAILURE: Authorino validated the JWT but response header # CEL expression hasn't stabilized (X-MaaS-Username empty) if (response.status_code == 403 and not response.text.strip()) \ or response.status_code == 401 \ or (response.status_code == 500 and "AUTH_FAILURE" in response.text): if attempt < retries: log.info("OIDC API key mint got %d (attempt %d/%d), retrying in %ds...", response.status_code, attempt, retries, delay) time.sleep(delay) continue break > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 500 {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"003"} E assert 500 in (200, 201) E + where 500 = <Response [500]>.status_code test/e2e/tests/test_external_oidc.py:195: AssertionErrorself = <test_external_oidc.TestOIDCMultiTenant object at 0x7f3a57148f70> maas_api_base_url = 'https://maas.apps.04faa770-22fb-475a-b4e9-f5a176fc4161.prod.konfluxeaas.com/maas-api' def test_tenant_a_users_are_isolated(self, maas_api_base_url: str): """Different tenant-a users can each mint their own API keys independently.""" alice_token = _request_oidc_token(username="alice_lead", password="letmein") bob_token = _request_oidc_token(username="bob_sre", password="letmein") alice_key = _create_oidc_api_key( maas_api_base_url, alice_token, name=f"e2e-alice-iso-{uuid.uuid4().hex[:8]}" ) > bob_key = _create_oidc_api_key( maas_api_base_url, bob_token, name=f"e2e-bob-iso-{uuid.uuid4().hex[:8]}" ) test/e2e/tests/test_external_oidc.py:488: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ maas_api_base_url = 'https://maas.apps.04faa770-22fb-475a-b4e9-f5a176fc4161.prod.konfluxeaas.com/maas-api' oidc_token = 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJFcnBNRklKVjJtU2RoYVNoenFzdzlwdWF5YlRFWXpCbzNFRzl1cVpzS3pRIn0.eyJle...hjMYQmxZPswnffoNTQamcGEKxVw2Rmq5m45eYDbijieeq35hNdm8JP_xmd9kAolIdB8UjbGmR-2vLF59ur_NixQbyRom7ChydDNZrr-4vy3csrG2jQDaWg' name = 'e2e-bob-iso-f5e1ba4c', subscription = None def _create_oidc_api_key( maas_api_base_url: str, oidc_token: str, name: str | None = None, subscription: str | None = None, ) -> dict: """Mint a MaaS API key using an OIDC bearer token. Retries on empty 403 (gateway propagation delay) and 401 (Authorino may still be loading the OIDC JWKS keys from Keycloak). Args: maas_api_base_url: MaaS API base URL oidc_token: OIDC access token name: Optional key name (auto-generated if omitted) subscription: Optional subscription to bind at mint time Returns: Full response body dict (includes 'key', 'id', 'subscription', etc.) """ body: dict = {"name": name or f"e2e-oidc-{uuid.uuid4().hex[:8]}"} if subscription: body["subscription"] = subscription retries, delay = 6, 5 for attempt in range(1, retries + 1): response = requests.post( f"{maas_api_base_url}/v1/api-keys", headers={"Authorization": f"Bearer {oidc_token}", "Content-Type": "application/json"}, json=body, timeout=30, verify=TLS_VERIFY, ) # Empty 403: gateway hasn't loaded AuthPolicy yet # 401: Authorino may still be fetching OIDC JWKS keys # 500 AUTH_FAILURE: Authorino validated the JWT but response header # CEL expression hasn't stabilized (X-MaaS-Username empty) if (response.status_code == 403 and not response.text.strip()) \ or response.status_code == 401 \ or (response.status_code == 500 and "AUTH_FAILURE" in response.text): if attempt < retries: log.info("OIDC API key mint got %d (attempt %d/%d), retrying in %ds...", response.status_code, attempt, retries, delay) time.sleep(delay) continue break > assert response.status_code in (200, 201), ( f"OIDC API key mint failed: {response.status_code} {response.text}" ) E AssertionError: OIDC API key mint failed: 500 {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"003"} E assert 500 in (200, 201) E + where 500 = <Response [500]>.status_code test/e2e/tests/test_external_oidc.py:195: AssertionErrorself = <test_external_oidc.TestOIDCAPIKeyLifecycle object at 0x7f3a57148fd0> maas_api_base_url = 'https://maas.apps.04faa770-22fb-475a-b4e9-f5a176fc4161.prod.konfluxeaas.com/maas-api' def test_create_and_revoke_api_key(self, maas_api_base_url: str): """Full create → revoke lifecycle with OIDC token.""" token = _request_oidc_token() # Create key_data = _create_oidc_api_key(maas_api_base_url, token, name=f"e2e-revoke-{uuid.uuid4().hex[:8]}") key_id = key_data["id"] # Revoke response = requests.delete( f"{maas_api_base_url}/v1/api-keys/{key_id}", headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, timeout=30, verify=TLS_VERIFY, ) > assert response.status_code in (200, 204), ( f"API key revocation failed: {response.status_code} {response.text}" ) E AssertionError: API key revocation failed: 500 {"error":"Exception thrown while generating token","exceptionCode":"AUTH_FAILURE","refId":"003"} E assert 500 in (200, 204) E + where 500 = <Response [500]>.status_code test/e2e/tests/test_external_oidc.py:521: AssertionError