{"level":"info","timestamp":"2026-07-03T19:24:02.760Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"ed220158-6c73-4145-b53b-50bb60e790c3","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:02.760Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"ed220158-6c73-4145-b53b-50bb60e790c3","keyPrefix":"sk-oai-NYobsH12r7oT...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:02 | 201 | 1.584621ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:02 | 200 | 283.665µs | 44.212.242.249 | GET "/v1/api-keys/ed220158-6c73-4145-b53b-50bb60e790c3" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:02 | 200 | 2.934747ms | 44.212.242.249 | POST "/v1/api-keys/search" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:02.856Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"ed220158-6c73-4145-b53b-50bb60e790c3"} {"level":"info","timestamp":"2026-07-03T19:24:02.856Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"ed220158-6c73-4145-b53b-50bb60e790c3","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"} [GIN] 2026/07/03 - 19:24:02 | 200 | 1.683642ms | 44.212.242.249 | DELETE "/v1/api-keys/ed220158-6c73-4145-b53b-50bb60e790c3" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:02.891Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"id":"bbdda45e-c2fe-416d-97bb-9e6279cc7a08","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:02.891Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"bbdda45e-c2fe-416d-97bb-9e6279cc7a08","keyPrefix":"sk-oai-VVfshZikyxZD...","username":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:02 | 201 | 1.601565ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"warn","timestamp":"2026-07-03T19:24:02.921Z","caller":"api_keys/handler.go:140","message":"Unauthorized API key access attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"bbdda45e-c2fe-416d-97bb-9e6279cc7a08"} [GIN] 2026/07/03 - 19:24:02 | 404 | 309.693µs | 44.212.242.249 | GET "/v1/api-keys/bbdda45e-c2fe-416d-97bb-9e6279cc7a08" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"warn","timestamp":"2026-07-03T19:24:02.952Z","caller":"api_keys/handler.go:347","message":"Unauthorized API key revocation attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"bbdda45e-c2fe-416d-97bb-9e6279cc7a08"} [GIN] 2026/07/03 - 19:24:02 | 404 | 337.201µs | 44.212.242.249 | DELETE "/v1/api-keys/bbdda45e-c2fe-416d-97bb-9e6279cc7a08" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:02.987Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1b96c587-0024-4f63-b3de-e129a80c5010","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:02.987Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"1b96c587-0024-4f63-b3de-e129a80c5010","keyPrefix":"sk-oai-1Fr1mHAgHrZ1...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:02 | 201 | 1.550339ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.018Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a8df0851-8cf3-4ea9-a09a-2088de3c758e","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.018Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"a8df0851-8cf3-4ea9-a09a-2088de3c758e","keyPrefix":"sk-oai-VjSJrDEvAJdY...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.476891ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.052Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a33cc83a-3744-4b95-945f-15d1d20cf0c0","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.052Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"a33cc83a-3744-4b95-945f-15d1d20cf0c0","keyPrefix":"sk-oai-x9Ekc2doV9jB...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.456524ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 279.793µs | 44.212.242.249 | GET "/v1/api-keys/1b96c587-0024-4f63-b3de-e129a80c5010" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.114Z","caller":"api_keys/store_postgres.go:467","message":"Revoked all keys for user","count":7,"user":"system:serviceaccount:default:tester-regular-user"} {"level":"info","timestamp":"2026-07-03T19:24:03.114Z","caller":"api_keys/handler.go:636","message":"Bulk revoked API keys","count":7,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:03 | 200 | 2.040987ms | 44.212.242.249 | POST "/v1/api-keys/bulk-revoke" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 288.336µs | 44.212.242.249 | GET "/v1/api-keys/1b96c587-0024-4f63-b3de-e129a80c5010" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 274.357µs | 44.212.242.249 | GET "/v1/api-keys/a8df0851-8cf3-4ea9-a09a-2088de3c758e" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 269.997µs | 44.212.242.249 | GET "/v1/api-keys/a33cc83a-3744-4b95-945f-15d1d20cf0c0" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"warn","timestamp":"2026-07-03T19:24:03.237Z","caller":"api_keys/handler.go:613","message":"Unauthorized bulk revoke attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","targetUser":"someotheruser"} [GIN] 2026/07/03 - 19:24:03 | 403 | 91.515µs | 44.212.242.249 | POST "/v1/api-keys/bulk-revoke" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.272Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e351c1de-6024-4e88-9e60-2c8ccc62ae92","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.272Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"e351c1de-6024-4e88-9e60-2c8ccc62ae92","keyPrefix":"sk-oai-HCwbHW7D6FYm...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.560742ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 281.252µs | 44.212.242.249 | GET "/v1/api-keys/e351c1de-6024-4e88-9e60-2c8ccc62ae92" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.334Z","caller":"api_keys/store_postgres.go:467","message":"Revoked all keys for user","count":1,"user":"system:serviceaccount:default:tester-regular-user"} {"level":"info","timestamp":"2026-07-03T19:24:03.334Z","caller":"api_keys/handler.go:636","message":"Bulk revoked API keys","count":1,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"} [GIN] 2026/07/03 - 19:24:03 | 200 | 1.513524ms | 44.212.242.249 | POST "/v1/api-keys/bulk-revoke" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.369Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fdbe4f46-a92c-45b7-bb2d-a6126f3d10f3","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.369Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"fdbe4f46-a92c-45b7-bb2d-a6126f3d10f3","keyPrefix":"sk-oai-DceX8hKjFgNr...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.594351ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.402Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e97df646-48cd-4f78-8d3a-fd17edcd6556","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.403Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"e97df646-48cd-4f78-8d3a-fd17edcd6556","keyPrefix":"sk-oai-QZUtq9RBmn5A...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.449038ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"error","timestamp":"2026-07-03T19:24:03.434Z","caller":"api_keys/handler.go:231","message":"Failed to create API key","error":"requested expiration (4320h0m0s) exceeds maximum allowed (90 days): expiration exceeds maximum allowed","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:231\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:123\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/03 - 19:24:03 | 400 | 139.095µs | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.467Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"4bb65756-450f-49c3-8b5b-84b095cda42a","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.468Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"4bb65756-450f-49c3-8b5b-84b095cda42a","keyPrefix":"sk-oai-1KMAyEJ6xLGP...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.640912ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.498Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"82361b1c-b3b1-4cc6-a501-1cbb072def4e","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.498Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"82361b1c-b3b1-4cc6-a501-1cbb072def4e","keyPrefix":"sk-oai-184DpculBSvc...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.503853ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:03.533Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1194802b-88c4-4675-ba40-17dc031fa582","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.533Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"1194802b-88c4-4675-ba40-17dc031fa582","keyPrefix":"sk-oai-kLo2mNntC9JR...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.542605ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 160.421µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:24:03 | 200 | 18.425132ms | 44.212.242.249 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 387.702µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:03 | 200 | 176.686µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:24:03 | 200 | 86.196µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" {"level":"info","timestamp":"2026-07-03T19:24:03.708Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"18487eb2-95dd-4bab-92e4-deb26e178427","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:03.708Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"18487eb2-95dd-4bab-92e4-deb26e178427","keyPrefix":"sk-oai-6QwDWxzqtziO...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:03 | 201 | 1.621742ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:03 | 200 | 309.315µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:03 | 200 | 177.644µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-03T19:24:03.772Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"18487eb2-95dd-4bab-92e4-deb26e178427"} {"level":"info","timestamp":"2026-07-03T19:24:03.772Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"18487eb2-95dd-4bab-92e4-deb26e178427","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:03 | 200 | 1.663061ms | 44.212.242.249 | DELETE "/v1/api-keys/18487eb2-95dd-4bab-92e4-deb26e178427" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:05 | 200 | 43.239µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:05 | 200 | 427.822µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:05 | 200 | 366.408µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:05 | 200 | 246.235µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-03T19:24:05.874Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"35dd6c6e-2b66-4c6e-85a9-7269b7d0e986","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:05.874Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"35dd6c6e-2b66-4c6e-85a9-7269b7d0e986","keyPrefix":"sk-oai-J07S6jIGgl6i...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:05 | 201 | 1.539283ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:05.904Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"35dd6c6e-2b66-4c6e-85a9-7269b7d0e986"} {"level":"info","timestamp":"2026-07-03T19:24:05.904Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"35dd6c6e-2b66-4c6e-85a9-7269b7d0e986","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:05 | 200 | 1.725808ms | 44.212.242.249 | DELETE "/v1/api-keys/35dd6c6e-2b66-4c6e-85a9-7269b7d0e986" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:05 | 404 | 504.787µs | 44.212.242.249 | DELETE "/v1/api-keys/35dd6c6e-2b66-4c6e-85a9-7269b7d0e986" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:05 | 404 | 245.037µs | 44.212.242.249 | DELETE "/v1/api-keys/nonexistent-uuid-12345" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.004Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e1659ad5-806d-4380-9637-d7d80fa4b187","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.004Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"e1659ad5-806d-4380-9637-d7d80fa4b187","keyPrefix":"sk-oai-56giGRMsCihB...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.553152ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.032Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"e1659ad5-806d-4380-9637-d7d80fa4b187"} {"level":"info","timestamp":"2026-07-03T19:24:06.032Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"e1659ad5-806d-4380-9637-d7d80fa4b187","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.623488ms | 44.212.242.249 | DELETE "/v1/api-keys/e1659ad5-806d-4380-9637-d7d80fa4b187" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.064Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"02623d6c-4ed9-4ef3-8bc8-3289ff1c1397","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.064Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"02623d6c-4ed9-4ef3-8bc8-3289ff1c1397","keyPrefix":"sk-oai-1M2YAbMxcWfW...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.413712ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:06 | 200 | 360.641µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:06 | 200 | 411.695µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:06 | 200 | 231.737µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-03T19:24:06.160Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2f87334d-27ee-45a5-83f4-343d4b7d16ff","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.160Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"2f87334d-27ee-45a5-83f4-343d4b7d16ff","keyPrefix":"sk-oai-HF3pvXAfWg9Y...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.482289ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.189Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"90e72495-c316-46f9-b894-666b890ea536","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.189Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"90e72495-c316-46f9-b894-666b890ea536","keyPrefix":"sk-oai-sD1X8THCzPKd...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.317673ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.220Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1939c71d-31ea-4910-ab84-fc16a337a5d5","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.220Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"1939c71d-31ea-4910-ab84-fc16a337a5d5","keyPrefix":"sk-oai-F3rnQXYXJhuK...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.480909ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.253Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"2f87334d-27ee-45a5-83f4-343d4b7d16ff"} {"level":"info","timestamp":"2026-07-03T19:24:06.253Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"2f87334d-27ee-45a5-83f4-343d4b7d16ff","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.667694ms | 44.212.242.249 | DELETE "/v1/api-keys/2f87334d-27ee-45a5-83f4-343d4b7d16ff" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.285Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"90e72495-c316-46f9-b894-666b890ea536"} {"level":"info","timestamp":"2026-07-03T19:24:06.285Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"90e72495-c316-46f9-b894-666b890ea536","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.549117ms | 44.212.242.249 | DELETE "/v1/api-keys/90e72495-c316-46f9-b894-666b890ea536" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.317Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"1939c71d-31ea-4910-ab84-fc16a337a5d5"} {"level":"info","timestamp":"2026-07-03T19:24:06.317Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"1939c71d-31ea-4910-ab84-fc16a337a5d5","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.581656ms | 44.212.242.249 | DELETE "/v1/api-keys/1939c71d-31ea-4910-ab84-fc16a337a5d5" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.353Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f6aa6a23-a127-4a4c-a4d1-2b82c6ef3d1b","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.353Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"f6aa6a23-a127-4a4c-a4d1-2b82c6ef3d1b","keyPrefix":"sk-oai-ArfnOFlqwJQ9...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.591919ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.383Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a8dc09be-2b84-439f-9fe6-0aafc586d987","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.383Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"a8dc09be-2b84-439f-9fe6-0aafc586d987","keyPrefix":"sk-oai-YuPw6kFLPQjf...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.505776ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.415Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f15739d0-48ce-460e-9103-a9b97ec74c42","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:06.415Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"f15739d0-48ce-460e-9103-a9b97ec74c42","keyPrefix":"sk-oai-1UjMeMR0GSDR...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.583415ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:06 | 200 | 335.097µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:06 | 200 | 204.295µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-03T19:24:06.482Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"f6aa6a23-a127-4a4c-a4d1-2b82c6ef3d1b"} {"level":"info","timestamp":"2026-07-03T19:24:06.482Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"f6aa6a23-a127-4a4c-a4d1-2b82c6ef3d1b","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.573802ms | 44.212.242.249 | DELETE "/v1/api-keys/f6aa6a23-a127-4a4c-a4d1-2b82c6ef3d1b" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.513Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"a8dc09be-2b84-439f-9fe6-0aafc586d987"} {"level":"info","timestamp":"2026-07-03T19:24:06.513Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"a8dc09be-2b84-439f-9fe6-0aafc586d987","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.711896ms | 44.212.242.249 | DELETE "/v1/api-keys/a8dc09be-2b84-439f-9fe6-0aafc586d987" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.545Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"f15739d0-48ce-460e-9103-a9b97ec74c42"} {"level":"info","timestamp":"2026-07-03T19:24:06.545Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"f15739d0-48ce-460e-9103-a9b97ec74c42","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:24:06 | 200 | 1.811634ms | 44.212.242.249 | DELETE "/v1/api-keys/f15739d0-48ce-460e-9103-a9b97ec74c42" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:06 | 200 | 383.473µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:06 | 200 | 329.965µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:06 | 200 | 382.501µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:24:06 | 200 | 39.325µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:24:06.877Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"0d4f27b7-e692-4f57-b188-f32bccc21308","ephemeral":true} {"level":"info","timestamp":"2026-07-03T19:24:06.877Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"0d4f27b7-e692-4f57-b188-f32bccc21308","keyPrefix":"sk-oai-eJ1Yd6JzkR7C...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.511661ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:06 | 200 | 3.280464ms | 44.212.242.249 | POST "/v1/api-keys/search" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:06 | 200 | 510.748µs | 44.212.242.249 | POST "/v1/api-keys/search" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:06.970Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"0939f48b-2ac7-4b66-9e17-d6f2330207ed","ephemeral":true} {"level":"info","timestamp":"2026-07-03T19:24:06.970Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"0939f48b-2ac7-4b66-9e17-d6f2330207ed","keyPrefix":"sk-oai-qYwGo0OQWWFf...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true} [GIN] 2026/07/03 - 19:24:06 | 201 | 1.547799ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:07.338Z","caller":"api_keys/service.go:473","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/07/03 - 19:24:07 | 200 | 766.83µs | ::1 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/07/03 - 19:24:07 | 200 | 270.566µs | 44.212.242.249 | GET "/v1/api-keys/0939f48b-2ac7-4b66-9e17-d6f2330207ed" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:24:08.088Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"74635da0-fd64-4c97-8b4a-7f27f79b406d","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:08.088Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"74635da0-fd64-4c97-8b4a-7f27f79b406d","keyPrefix":"sk-oai-Ah9jQTCgspdA...","username":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:08 | 201 | 1.692434ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:10 | 200 | 39.224µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:15 | 200 | 37.098µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:16 | 200 | 38.203µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:20 | 200 | 32.22µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:25 | 200 | 35.15µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:26 | 200 | 35.092µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:30 | 200 | 32.544µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:35 | 200 | 35.206µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:36 | 200 | 55.203µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:40 | 200 | 42.807µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:45 | 200 | 34.542µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:46 | 200 | 40.629µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:50 | 200 | 37.053µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:24:53.154Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"a547409a-e444-4728-a395-97467ae51402","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:24:53.154Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"a547409a-e444-4728-a395-97467ae51402","keyPrefix":"sk-oai-Nfw9PVwmpjFQ...","username":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:24:53 | 201 | 2.459566ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:24:55 | 200 | 33.974µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:24:56 | 200 | 33.325µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:00 | 200 | 32.226µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:05 | 200 | 37.978µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:06 | 200 | 35.47µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:10 | 200 | 33.864µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:15 | 200 | 34.158µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:16 | 200 | 34.354µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:20 | 200 | 33.109µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:25 | 200 | 34.156µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:26 | 200 | 33.492µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:30 | 200 | 42.988µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:35 | 200 | 33.307µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:36 | 200 | 32.752µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-07-03T19:25:38.385Z","caller":"api_keys/service.go:174","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-failed-sa","requestedSubscription":"e2e-apikey-failed-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-07-03T19:25:38.385Z","caller":"api_keys/handler.go:231","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:231\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:123\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/03 - 19:25:38 | 403 | 319.866µs | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:25:40 | 200 | 34.715µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:45 | 200 | 35.231µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:46 | 200 | 33.227µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:50 | 200 | 33.202µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:55 | 200 | 34.89µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:25:56 | 200 | 36.423µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:00 | 200 | 37.396µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:05 | 200 | 31.784µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:06 | 200 | 42.595µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:10 | 200 | 32.841µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:15 | 200 | 32.539µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:16 | 200 | 42.438µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:20 | 200 | 32.722µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:26:23.601Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"5709cb09-5607-49c8-b05d-3c13103b4b5f","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:26:23.601Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"5709cb09-5607-49c8-b05d-3c13103b4b5f","keyPrefix":"sk-oai-wvUC8WQdW7e7...","username":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:26:23 | 201 | 2.937843ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:26:25 | 200 | 33.165µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:26 | 200 | 30.907µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:30 | 200 | 35.842µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:35 | 200 | 31.604µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:36 | 200 | 32.776µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:40 | 200 | 32.643µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:45 | 200 | 40.187µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:46 | 200 | 38.567µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:50 | 200 | 32.886µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:55 | 200 | 41.219µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:26:56 | 200 | 36.268µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-07-03T19:26:59.776Z","caller":"api_keys/service.go:174","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-unreconciled-sa","requestedSubscription":"e2e-apikey-unreconciled-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-07-03T19:26:59.776Z","caller":"api_keys/handler.go:231","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:231\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:123\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/03 - 19:26:59 | 400 | 314.168µs | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:27:00 | 200 | 42.127µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:05 | 200 | 32.518µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:06 | 200 | 34.995µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:10 | 200 | 31.37µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:15 | 200 | 32.295µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:16 | 200 | 30.895µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:20 | 200 | 35.259µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:25 | 200 | 37.587µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:26 | 200 | 31.411µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:30 | 200 | 33.723µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:35 | 200 | 33.754µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:36 | 200 | 32.62µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:27:36.781Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-c333e18b","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"b22d9635-dec9-4ac0-9fee-4975c111d8c1","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:27:36.781Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"b22d9635-dec9-4ac0-9fee-4975c111d8c1","keyPrefix":"sk-oai-1LDBKNihI7PC...","username":"system:serviceaccount:llm:e2e-filter-sa-c333e18b","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:27:36 | 201 | 1.768844ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:27:36.816Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-c333e18b","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"f0128ba9-53e4-4c63-bd65-4dd15c5630a2","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:27:36.816Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"f0128ba9-53e4-4c63-bd65-4dd15c5630a2","keyPrefix":"sk-oai-1NG5cYSCw4i9...","username":"system:serviceaccount:llm:e2e-filter-sa-c333e18b","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:27:36 | 201 | 1.659977ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:27:36.849Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-c333e18b","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"fbf2f5b2-394a-4f97-b6c5-4d4817705a83","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:27:36.849Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"fbf2f5b2-394a-4f97-b6c5-4d4817705a83","keyPrefix":"sk-oai-ZkLkpAGalV1V...","username":"system:serviceaccount:llm:e2e-filter-sa-c333e18b","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:27:36 | 201 | 1.576933ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:27:36 | 200 | 3.381174ms | 44.212.242.249 | POST "/v1/api-keys/search" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:27:36.918Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"b22d9635-dec9-4ac0-9fee-4975c111d8c1"} {"level":"info","timestamp":"2026-07-03T19:27:36.918Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"b22d9635-dec9-4ac0-9fee-4975c111d8c1","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-c333e18b"} [GIN] 2026/07/03 - 19:27:36 | 200 | 1.889298ms | 44.212.242.249 | DELETE "/v1/api-keys/b22d9635-dec9-4ac0-9fee-4975c111d8c1" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:27:36.953Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"f0128ba9-53e4-4c63-bd65-4dd15c5630a2"} {"level":"info","timestamp":"2026-07-03T19:27:36.953Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"f0128ba9-53e4-4c63-bd65-4dd15c5630a2","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-c333e18b"} [GIN] 2026/07/03 - 19:27:36 | 200 | 1.573077ms | 44.212.242.249 | DELETE "/v1/api-keys/f0128ba9-53e4-4c63-bd65-4dd15c5630a2" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:27:36.984Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"fbf2f5b2-394a-4f97-b6c5-4d4817705a83"} {"level":"info","timestamp":"2026-07-03T19:27:36.984Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"fbf2f5b2-394a-4f97-b6c5-4d4817705a83","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-c333e18b"} [GIN] 2026/07/03 - 19:27:36 | 200 | 1.772443ms | 44.212.242.249 | DELETE "/v1/api-keys/fbf2f5b2-394a-4f97-b6c5-4d4817705a83" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:27:40 | 200 | 37.743µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:45 | 200 | 38.229µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:46 | 200 | 34.148µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:50 | 200 | 32.014µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:55 | 200 | 33.371µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:27:56 | 200 | 45.468µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:00 | 200 | 32.803µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:05 | 200 | 33.925µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:06 | 200 | 43.564µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:10 | 200 | 32.235µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:15 | 200 | 34.457µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:16 | 200 | 35.204µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:20 | 200 | 39.932µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:25 | 200 | 33.462µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:26 | 200 | 32.61µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:30 | 200 | 30.715µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:35 | 200 | 31.782µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:36 | 200 | 33.742µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:40 | 200 | 33.093µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:28:41.859Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"769afde3-68fd-495c-b12a-05a8a0d462b1","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:28:41.859Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"769afde3-68fd-495c-b12a-05a8a0d462b1","keyPrefix":"sk-oai-1EwwCdWCRQ7y...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:28:41 | 201 | 1.754191ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:28:41.898Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"8e08ff97-c27a-4ac8-bedf-ceeeac9453cf","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:28:41.898Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"8e08ff97-c27a-4ac8-bedf-ceeeac9453cf","keyPrefix":"sk-oai-tCfxBRTgfScz...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:28:41 | 201 | 1.632595ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:28:41 | 200 | 3.434041ms | 44.212.242.249 | POST "/v1/api-keys/search" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:28:41.972Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"769afde3-68fd-495c-b12a-05a8a0d462b1"} {"level":"info","timestamp":"2026-07-03T19:28:41.972Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"769afde3-68fd-495c-b12a-05a8a0d462b1","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:28:41 | 200 | 1.67181ms | 44.212.242.249 | DELETE "/v1/api-keys/769afde3-68fd-495c-b12a-05a8a0d462b1" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:28:42.014Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"8e08ff97-c27a-4ac8-bedf-ceeeac9453cf"} {"level":"info","timestamp":"2026-07-03T19:28:42.014Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"8e08ff97-c27a-4ac8-bedf-ceeeac9453cf","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:28:42 | 200 | 1.627001ms | 44.212.242.249 | DELETE "/v1/api-keys/8e08ff97-c27a-4ac8-bedf-ceeeac9453cf" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:28:42.053Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2f1594bd-0bc2-4515-96bd-3507be5b5ce2","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:28:42.053Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"2f1594bd-0bc2-4515-96bd-3507be5b5ce2","keyPrefix":"sk-oai-CWnqMfU5aN1j...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:28:42 | 201 | 1.585468ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:28:42 | 200 | 597.696µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:28:42 | 200 | 229.306µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-03T19:28:42.135Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"c2954d37-5349-4dec-af13-699b73522b0b","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:28:42.135Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"c2954d37-5349-4dec-af13-699b73522b0b","keyPrefix":"sk-oai-1K4GQXvfr2Dt...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:28:42 | 201 | 1.614242ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:28:42 | 200 | 329.473µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:28:42 | 200 | 276.234µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-03T19:28:42.206Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3772099f-948c-4620-bd7a-feaf57767ba0","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:28:42.206Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"3772099f-948c-4620-bd7a-feaf57767ba0","keyPrefix":"sk-oai-1LSBHLjZPrEI...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:28:42 | 201 | 1.583593ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:28:45 | 200 | 33.905µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:46 | 200 | 31.972µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:47 | 200 | 2.017485ms | 10.132.0.37 | POST "/internal/v1/api-keys/validate" {"level":"info","timestamp":"2026-07-03T19:28:47.279Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"30dc3bdf-96b9-4152-93be-2605bf8e7402","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:28:47.279Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"30dc3bdf-96b9-4152-93be-2605bf8e7402","keyPrefix":"sk-oai-1b7EB6flSfb2...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:28:47 | 201 | 1.568361ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:28:47 | 200 | 365.747µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:28:47 | 200 | 327.604µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:28:50 | 200 | 34.019µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:55 | 200 | 41.215µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:28:56 | 200 | 51.201µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:00 | 200 | 58.644µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:05 | 200 | 46.634µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:06 | 200 | 47.339µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:10 | 200 | 43.401µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:15 | 200 | 43.293µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:16 | 200 | 39.852µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:20 | 200 | 40.916µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:25 | 200 | 41.179µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:26 | 200 | 42.347µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:30 | 200 | 38.977µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:35 | 200 | 42.866µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:29:36.666Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"0b50e9e2-27bc-45a5-9720-7bdf29fce982","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:29:36.666Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"0b50e9e2-27bc-45a5-9720-7bdf29fce982","keyPrefix":"sk-oai-ZwvKPkLo9KJB...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:29:36 | 201 | 10.16812ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:29:36 | 200 | 38.656µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:36 | 200 | 1.009219ms | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:29:36 | 200 | 260.662µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:29:36 | 200 | 491.291µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:29:36 | 200 | 288.066µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:29:36 | 200 | 436.18µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:29:36 | 200 | 251.479µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:29:36 | 200 | 504.589µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:29:36 | 200 | 363.259µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:29:40 | 200 | 39.897µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:45 | 200 | 46.959µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:46 | 200 | 41.51µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:29:48.784Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"4d2a9efe-f694-4299-9baa-96ae22c1c7d2","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:29:48.784Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"4d2a9efe-f694-4299-9baa-96ae22c1c7d2","keyPrefix":"sk-oai-1Y1t3FMFCubO...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:29:48 | 201 | 1.876795ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:29:48 | 200 | 433.343µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:29:48 | 200 | 274.188µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:29:48 | 200 | 477.565µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:29:48 | 200 | 262.838µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:29:50 | 200 | 39.58µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:55 | 200 | 44.346µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:29:56 | 200 | 42.162µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:00 | 200 | 41.865µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:30:04.430Z","caller":"api_keys/service.go:473","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/07/03 - 19:30:04 | 200 | 786.736µs | 10.134.0.54 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/07/03 - 19:30:05 | 200 | 39.264µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:06 | 200 | 40.019µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:30:08.241Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"06e85a6a-e5c3-4329-a059-16cf39742578","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:30:08.241Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"06e85a6a-e5c3-4329-a059-16cf39742578","keyPrefix":"sk-oai-1KViXtpbfUkk...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:30:08 | 201 | 1.772965ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:30:08.271Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"06e85a6a-e5c3-4329-a059-16cf39742578"} {"level":"info","timestamp":"2026-07-03T19:30:08.272Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"06e85a6a-e5c3-4329-a059-16cf39742578","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:30:08 | 200 | 2.15104ms | 44.212.242.249 | DELETE "/v1/api-keys/06e85a6a-e5c3-4329-a059-16cf39742578" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:30:08.309Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2871e395-fe13-4744-b115-517c2061e68c","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:30:08.309Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"2871e395-fe13-4744-b115-517c2061e68c","keyPrefix":"sk-oai-1YAICP2ZFBQL...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:30:08 | 201 | 1.810868ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-03T19:30:08.342Z","caller":"api_keys/store_postgres.go:487","message":"Revoked API key","id":"2871e395-fe13-4744-b115-517c2061e68c"} {"level":"info","timestamp":"2026-07-03T19:30:08.342Z","caller":"api_keys/handler.go:368","message":"Revoked API key","keyId":"2871e395-fe13-4744-b115-517c2061e68c","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/07/03 - 19:30:08 | 200 | 2.011662ms | 44.212.242.249 | DELETE "/v1/api-keys/2871e395-fe13-4744-b115-517c2061e68c" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"warn","timestamp":"2026-07-03T19:30:08.377Z","caller":"api_keys/service.go:174","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:default:tester-regular-user","requestedSubscription":"e2e-no-such-subscription-86a7357869de429ea58171605125ed1f","error":"requested subscription not found"} {"level":"error","timestamp":"2026-07-03T19:30:08.377Z","caller":"api_keys/handler.go:231","message":"Failed to create API key","error":"requested subscription not found","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:231\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:123\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/03 - 19:30:08 | 400 | 394.237µs | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:30:10 | 200 | 32.78µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:15 | 200 | 34.598µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:16 | 200 | 35.634µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:20 | 200 | 54.408µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:25 | 200 | 31.7µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:26 | 200 | 33.786µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:30 | 200 | 34.075µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:35 | 200 | 34.11µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:36 | 200 | 42.621µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:38 | 200 | 434.309µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:30:38 | 200 | 377.287µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:30:38 | 200 | 410.541µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:30:38 | 200 | 361.199µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:30:40 | 200 | 32.64µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:42 | 200 | 434.165µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:30:42 | 200 | 353.241µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:30:45 | 200 | 33.485µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:46 | 200 | 34.36µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:50 | 200 | 35.531µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:55 | 200 | 34.243µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:30:56 | 200 | 42.779µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:00 | 200 | 32.782µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:05 | 200 | 40.637µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:06 | 200 | 34.477µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:10 | 200 | 31.201µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:15 | 200 | 32.114µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:16 | 200 | 35.358µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:31:17.229Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3fdae868-96fa-4dea-917b-ddf80b8b69cd","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:31:17.229Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"3fdae868-96fa-4dea-917b-ddf80b8b69cd","keyPrefix":"sk-oai-177oqR5Zz7I0...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:31:17 | 201 | 1.883415ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:31:17 | 200 | 388.104µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:31:17 | 200 | 428.275µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:31:20 | 200 | 34.087µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:25 | 200 | 38.623µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:26 | 200 | 38.186µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:30 | 200 | 57.769µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:35 | 200 | 39.483µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:36 | 200 | 38.851µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:40 | 200 | 49.739µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:45 | 200 | 41.536µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:46 | 200 | 47.752µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:50 | 200 | 38.286µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:55 | 200 | 48.44µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:31:56 | 200 | 55.94µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:00 | 200 | 51.094µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:05 | 200 | 40.226µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:06 | 200 | 38.379µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:10 | 200 | 38.173µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:32:15.548Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"111b5368-d40d-4e75-aa00-2b201b140d6e","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:32:15.548Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"111b5368-d40d-4e75-aa00-2b201b140d6e","keyPrefix":"sk-oai-AFwDu57DVjCU...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:32:15 | 201 | 2.726058ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:32:15 | 200 | 372.167µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:32:15 | 200 | 408.338µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:32:15 | 200 | 405.116µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:32:15 | 200 | 286.26µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:32:15 | 200 | 322.445µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:32:15 | 200 | 319.346µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:32:15 | 200 | 31.958µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:15 | 200 | 302.909µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:32:15 | 200 | 353.602µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:32:15 | 200 | 345.392µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:32:15 | 200 | 342.567µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:32:16 | 200 | 38.04µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:20 | 200 | 41.307µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:25 | 200 | 38.936µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:26 | 200 | 43.047µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:30 | 200 | 40.047µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:35 | 200 | 44µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:36 | 200 | 37.51µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:40 | 200 | 41.642µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:45 | 200 | 44.875µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:46 | 200 | 42.089µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:50 | 200 | 447.484µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:32:50 | 200 | 333.886µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:32:50 | 200 | 48.76µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:55 | 200 | 34.161µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:32:56 | 200 | 34.476µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-07-03T19:32:58.762Z","caller":"api_keys/service.go:174","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:default:tester-regular-user","requestedSubscription":"e2e-premium-sa-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-07-03T19:32:58.762Z","caller":"api_keys/handler.go:231","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:231\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:123\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/03 - 19:32:58 | 400 | 418.038µs | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:33:00 | 200 | 32.333µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:05 | 200 | 34.843µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:06 | 200 | 33.737µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:10 | 200 | 33.908µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:11 | 200 | 478.596µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:33:11 | 200 | 410.637µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:33:15 | 200 | 33.394µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:16 | 200 | 40.593µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:19 | 200 | 476.124µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:33:19 | 200 | 336.35µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:33:20 | 200 | 32.548µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:25 | 200 | 33.599µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:26 | 200 | 32.32µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:30 | 200 | 31.598µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:35 | 200 | 33.286µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:36 | 200 | 35.909µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:40 | 200 | 56.312µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:45 | 200 | 35.429µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:46 | 200 | 33.81µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:50 | 200 | 35.638µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:55 | 200 | 35.304µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:33:56 | 200 | 36.096µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:00 | 200 | 33.431µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:01 | 200 | 596.897µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:01 | 200 | 312.437µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:05 | 200 | 36.055µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:06 | 200 | 32.875µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:06 | 200 | 441.217µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:06 | 200 | 552.902µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:10 | 200 | 38.065µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:15 | 200 | 34.2µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:16 | 200 | 32.517µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:20 | 200 | 34.069µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-03T19:34:22.137Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-ordering-sa-003795","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"0bbf1d46-25a9-4c22-897a-a2849b7ee28e","ephemeral":false} {"level":"info","timestamp":"2026-07-03T19:34:22.137Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"0bbf1d46-25a9-4c22-897a-a2849b7ee28e","keyPrefix":"sk-oai-TtIuA4eqhJ5Y...","username":"system:serviceaccount:models-as-a-service:e2e-ordering-sa-003795","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/03 - 19:34:22 | 201 | 1.933739ms | 44.212.242.249 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/03 - 19:34:22 | 200 | 396.61µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:22 | 200 | 277.294µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:22 | 200 | 351.971µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:22 | 200 | 312.151µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:24 | 200 | 410.434µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:24 | 200 | 320.771µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:25 | 200 | 33.849µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:26 | 200 | 418.184µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:26 | 200 | 353.954µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:26 | 200 | 33.767µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:28 | 200 | 457.551µs | 10.132.0.37 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/03 - 19:34:28 | 200 | 367.938µs | 10.132.0.37 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/03 - 19:34:30 | 200 | 35.202µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:35 | 200 | 35.749µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:36 | 200 | 55.614µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:40 | 200 | 34.07µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:45 | 200 | 36.634µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:46 | 200 | 35.703µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:50 | 200 | 36.778µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:55 | 200 | 48.356µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:34:56 | 200 | 35.754µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:00 | 200 | 33.285µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:05 | 200 | 36.597µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:06 | 200 | 35.842µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:10 | 200 | 36.971µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:15 | 200 | 38.212µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:16 | 200 | 39.201µs | 10.134.0.2 | GET "/health" [GIN] 2026/07/03 - 19:35:20 | 200 | 38.161µs | 10.134.0.2 | GET "/health"