{"level":"info","timestamp":"2026-05-29T17:20:11.671Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"0771c7e9-3b3b-41b2-b028-2da1835c34fc","keyPrefix":"sk-oai-JfE4g5GPIlKL...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:20:11 | 201 |    1.684276ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:20:11 | 200 |     677.937µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:20:11 | 200 |     477.069µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:20:11 | 200 |     629.767µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:20:11 | 200 |     425.167µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:20:13 | 200 |      35.824µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:18 | 200 |      34.658µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:18 | 200 |      32.051µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:23 | 200 |       38.82µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:28 | 200 |      31.332µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:28 | 200 |      33.088µs |      10.132.0.2 | GET      "/health"
{"level":"warn","timestamp":"2026-05-29T17:20:29.436Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-user","requestedSubscription":"e2e-test-other-subscription","error":"access denied to requested subscription"}
{"level":"error","timestamp":"2026-05-29T17:20:29.436Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"access denied to requested subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/05/29 - 17:20:29 | 400 |     287.804µs |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:20:33 | 200 |      34.541µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:38 | 200 |       35.33µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:38 | 200 |      32.984µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:43 | 200 |      43.292µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:20:46.898Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"598bd09c-4496-4e71-ab93-553f2b027e56","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:20:46.898Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"598bd09c-4496-4e71-ab93-553f2b027e56","keyPrefix":"sk-oai-F8xVDmuI1mQG...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:20:46 | 201 |    10.46738ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:20:46 | 200 |    1.401675ms |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:20:46 | 200 |      364.37µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:20:48 | 200 |      36.559µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:48 | 200 |       33.29µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:53 | 200 |      32.367µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:20:55.988Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-auth-only","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"45606bff-9ac2-4af7-ae74-f9eee5d98fd6","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:20:55.988Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"45606bff-9ac2-4af7-ae74-f9eee5d98fd6","keyPrefix":"sk-oai-1Te05XFzaOxn...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-auth-only","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:20:55 | 201 |    2.078939ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:20:58 | 200 |      37.219µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:20:58 | 200 |      32.542µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:03 | 200 |      36.121µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:04 | 200 |    1.199771ms |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:21:04 | 200 |     358.622µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:21:08 | 200 |        38.1µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:08 | 200 |      33.308µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:13 | 200 |      32.714µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:18 | 200 |       34.03µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:18 | 200 |      32.365µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:21:21.628Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"eab1cf94-c684-48e4-8f8a-6fca50df99fb","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:21:21.628Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"eab1cf94-c684-48e4-8f8a-6fca50df99fb","keyPrefix":"sk-oai-1L1z05whk5GC...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:21:21 | 201 |    2.189617ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:21:21 | 200 |     542.231µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:21:21 | 200 |     231.323µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:21:23 | 200 |      59.998µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:28 | 200 |      35.332µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:28 | 200 |      33.545µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:33 | 200 |      35.663µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:38 | 200 |      39.167µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:38 | 200 |      51.634µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:43 | 200 |      42.268µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:48 | 200 |      39.584µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:48 | 200 |      38.033µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:53 | 200 |      41.115µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:58 | 200 |      38.093µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:21:58 | 200 |      40.494µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:03 | 200 |      39.846µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:08 | 200 |       38.24µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:08 | 200 |      50.704µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:13 | 200 |      45.107µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:18 | 200 |      40.053µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:18 | 200 |      40.727µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:23 | 200 |      37.583µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:28 | 200 |      40.265µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:28 | 200 |      38.277µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:33 | 200 |      37.848µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:38 | 200 |      41.388µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:38 | 200 |      41.179µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:43 | 200 |      39.694µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:48 | 200 |      48.996µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:48 | 200 |      42.814µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:53 | 200 |      42.891µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:58 | 200 |      38.069µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:22:58 | 200 |      41.664µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:03 | 200 |      38.638µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:08 | 200 |      38.361µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:08 | 200 |      38.697µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:13 | 200 |      40.198µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:18 | 200 |      42.298µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:18 | 200 |      40.842µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:23:21.481Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"16c4915a-2d91-48fa-a531-acc03e39ece9","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:23:21.481Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"16c4915a-2d91-48fa-a531-acc03e39ece9","keyPrefix":"sk-oai-VX9pl6G7X2QK...","username":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:23:21 | 201 |    3.560611ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:23:23 | 200 |      39.482µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:28 | 200 |      40.176µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:28 | 200 |      40.001µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:33 | 200 |      43.211µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:38 | 200 |      35.463µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:38 | 200 |      32.959µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:43 | 200 |      33.407µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:44 | 200 |    1.233422ms |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:23:44 | 200 |     150.732µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:23:45 | 404 |    1.109541ms |      100.64.0.2 | DELETE   "/v1/api-keys/e2e-trlp-test-key" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:23:48 | 200 |      39.551µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:48 | 200 |      33.429µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:53 | 200 |      32.963µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:58 | 200 |      32.665µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:23:58 | 200 |      41.917µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:03 | 200 |      32.622µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:08 | 200 |      34.335µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:08 | 200 |      32.313µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:13 | 200 |      48.258µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:18 | 200 |      33.419µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:18 | 200 |      34.378µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:23 | 200 |      37.181µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:28 | 200 |      37.081µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:28 | 200 |      35.398µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:33 | 200 |      32.684µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:24:37.866Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"7efed2b3-092e-4011-a837-ae0ebe8bd335","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:24:37.866Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"7efed2b3-092e-4011-a837-ae0ebe8bd335","keyPrefix":"sk-oai-DRljTWheyRVc...","username":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:24:37 | 201 |    1.872606ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:24:37 | 200 |     570.818µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:24:37 | 200 |     146.173µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:24:38 | 200 |      33.584µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:38 | 200 |      33.247µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:43 | 200 |      33.641µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:48 | 200 |      38.157µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:48 | 200 |      32.804µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:53 | 200 |      34.132µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:24:57.216Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"5cf9b0ea-5e22-48e9-9754-49463748fd73","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:24:57.216Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"5cf9b0ea-5e22-48e9-9754-49463748fd73","keyPrefix":"sk-oai-tsFLAohyTBwp...","username":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:24:57 | 201 |    2.099639ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:24:57 | 200 |     563.068µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:24:57 | 200 |     154.854µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:24:57 | 200 |     527.351µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:24:57 | 200 |     141.702µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:24:58 | 200 |      32.887µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:24:58 | 200 |      38.258µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:03 | 200 |      33.128µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:08 | 200 |      33.636µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:08 | 200 |      30.155µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:13 | 200 |      32.517µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:25:16.825Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"495ab6d3-7979-458a-b724-d35d2ef124fa","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:25:16.825Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"495ab6d3-7979-458a-b724-d35d2ef124fa","keyPrefix":"sk-oai-v6AlVJE5gry8...","username":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:25:16 | 201 |    2.259898ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:25:16 | 200 |     403.647µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:25:16 | 200 |     417.259µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:25:16 | 200 |     163.062µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:25:16 | 200 |   17.781382ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:25:18 | 200 |       41.95µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:18 | 200 |      32.951µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:23 | 200 |      34.212µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:28 | 200 |      34.434µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:28 | 200 |      38.906µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:33 | 200 |      38.209µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:36 | 200 |     160.733µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:25:36 | 200 |     147.422µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:25:36 | 200 |   22.387772ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:25:38 | 200 |       39.44µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:38 | 200 |      37.951µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:43 | 200 |      40.288µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:25:48.158Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"cecbed2f-d73b-40ca-b8ca-93d40472ad20","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:25:48.158Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"cecbed2f-d73b-40ca-b8ca-93d40472ad20","keyPrefix":"sk-oai-3oxIe8aWJdV2...","username":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:25:48 | 201 |   10.882158ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:25:48 | 200 |      39.356µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:48 | 200 |      38.623µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:53 | 200 |      38.804µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:58 | 200 |      65.309µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:25:58 | 200 |      38.712µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:03 | 200 |      38.755µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:08 | 200 |      61.677µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:08 | 200 |      65.548µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:13 | 200 |      61.959µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:18 | 200 |    1.361178ms |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:26:18 | 200 |     145.314µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:26:18 | 200 |     499.877µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:26:18 | 200 |     436.127µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:26:18 | 200 |     121.764µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:26:18 | 200 |   12.432038ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:26:18 | 200 |      40.548µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:18 | 200 |      37.828µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:23 | 200 |      40.107µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:28 | 200 |       39.84µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:28 | 200 |      38.411µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:33 | 200 |      43.018µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:35 | 200 |     133.663µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:26:35 | 200 |   12.929384ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:26:38 | 200 |      43.059µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:38 | 200 |      38.663µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:43 | 200 |      37.828µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:26:44.113Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"563513b6-ef9b-49a2-b471-94f9265e1dec","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:26:44.113Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"563513b6-ef9b-49a2-b471-94f9265e1dec","keyPrefix":"sk-oai-1IDuae6FgnYG...","username":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:26:44 | 201 |    3.585709ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:26:48 | 200 |      38.929µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:48 | 200 |      40.153µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:52 | 200 |      930.11µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:26:52 | 200 |     644.518µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:26:52 | 200 |     358.124µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:26:52 | 200 |   10.034556ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:26:52.747Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a5f1a8bb-e51b-4d76-88b8-5f84b6908649","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:26:52.747Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"a5f1a8bb-e51b-4d76-88b8-5f84b6908649","keyPrefix":"sk-oai-1HcrYN9aHkkT...","username":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:26:52 | 201 |    1.718392ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:26:53 | 200 |      43.808µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:58 | 200 |      41.356µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:26:58 | 200 |      47.796µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:00 | 200 |     818.564µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:00 | 200 |     352.246µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:00 | 200 |     115.097µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:00 | 200 |    6.621853ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:00 | 200 |     329.531µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:00 | 200 |     356.764µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:00 | 200 |     177.632µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:00 | 200 |    6.493394ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:27:01.874Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"6fd60c6f-cc43-46fe-8933-8cdc1c18db89","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:27:01.874Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"6fd60c6f-cc43-46fe-8933-8cdc1c18db89","keyPrefix":"sk-oai-AMWtGpttrXTa...","username":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:27:01 | 201 |    1.767409ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:03 | 200 |      40.457µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:08 | 200 |       53.59µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:08 | 200 |      47.721µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:09 | 200 |     672.511µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:09 | 200 |     440.118µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:09 | 200 |       238.9µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:09 | 200 |   10.188533ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:13 | 200 |      40.008µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:18 | 200 |      37.716µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:18 | 200 |       48.46µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:27:19.172Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"daa87470-4be5-4e3a-9a1e-31123f63d185","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:27:19.172Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"daa87470-4be5-4e3a-9a1e-31123f63d185","keyPrefix":"sk-oai-1YcFtGsfU8IQ...","username":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:27:19 | 201 |    2.129187ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:23 | 200 |      36.829µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:27 | 200 |     740.007µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:27 | 200 |     439.077µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:27 | 200 |     323.991µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:27 | 200 |     445.258µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:27 | 200 |     255.246µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:27 | 200 |   18.036975ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:28 | 200 |      37.537µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:28 | 200 |      46.232µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:33 | 200 |      42.536µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:38 | 200 |      39.858µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:38 | 200 |      38.622µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:27:40.730Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"c5f0bbfb-ff3c-459e-8057-d8d892eed3a1","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:27:40.730Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"c5f0bbfb-ff3c-459e-8057-d8d892eed3a1","keyPrefix":"sk-oai-1QdtTTqAnaBJ...","username":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:27:40 | 201 |     1.48453ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:43 | 200 |      54.415µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:48 | 200 |      42.307µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:48 | 200 |      644.54µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:48 | 200 |     392.854µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:48 | 200 |     207.503µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:48 | 200 |    9.667163ms |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:27:48 | 200 |     162.186µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:27:48 | 200 |   32.040725ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:27:48 | 200 |      26.865µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:53 | 200 |      38.871µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:58 | 200 |      39.664µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:27:58 | 200 |      40.406µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:03 | 200 |      41.724µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:07 | 200 |     163.553µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:28:07 | 200 |     155.552µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:28:07 | 200 |       133.9µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:28:07 | 200 |   34.948085ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:28:08 | 200 |      44.266µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:08 | 200 |       38.18µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:13 | 200 |      40.477µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:18 | 200 |      39.764µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:18 | 200 |      61.201µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:23 | 200 |      44.139µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:24 | 200 |     170.473µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:28:24 | 200 |   13.525463ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:28:28 | 200 |      38.544µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:28 | 200 |      32.338µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:33 | 200 |      38.741µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:28:35.912Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"dc1ebed5-a328-4915-b1dd-34dd7e59f5bc","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:28:35.912Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"dc1ebed5-a328-4915-b1dd-34dd7e59f5bc","keyPrefix":"sk-oai-aSwDi7DhLEH9...","username":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:28:35 | 201 |    1.635193ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:28:38 | 200 |       37.74µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:38 | 200 |      44.509µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:43 | 200 |      44.292µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:43 | 200 |     787.721µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:28:43 | 200 |     158.202µs |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:28:44.566Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"25680ba3-eea0-4d40-a4f2-177bc9d818b5","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:28:44.566Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"25680ba3-eea0-4d40-a4f2-177bc9d818b5","keyPrefix":"sk-oai-JqCaFbM5TpfO...","username":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:28:44 | 201 |     1.78343ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:28:48 | 200 |      40.782µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:48 | 200 |        40.1µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:52 | 200 |     797.972µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:28:52 | 200 |     398.265µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:28:52 | 200 |     129.437µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:28:52 | 200 |    9.738198ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:28:52.985Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"ba41f81c-d615-4ca3-8456-7dfb1b9fc3ad","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:28:52.985Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"ba41f81c-d615-4ca3-8456-7dfb1b9fc3ad","keyPrefix":"sk-oai-EA3OTQhh2i0w...","username":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:28:52 | 201 |    1.780555ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:28:53 | 200 |      42.982µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:58 | 200 |      39.872µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:28:58 | 200 |      40.555µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:01 | 200 |     996.365µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:29:01 | 200 |     682.864µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:29:01 | 200 |     351.096µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:29:01 | 200 |   10.616194ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:29:01.802Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"c60649b4-9ac8-429d-bfcc-548147db7736","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:29:01.802Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"c60649b4-9ac8-429d-bfcc-548147db7736","keyPrefix":"sk-oai-aC2eLx7wQ0HK...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:29:01 | 201 |    1.907748ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:29:03 | 200 |      38.802µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:08 | 200 |      41.332µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:08 | 200 |      41.598µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:09 | 200 |      953.28µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:29:09 | 200 |     480.164µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:29:09 | 200 |     256.566µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:29:09 | 200 |    9.183859ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:29:13 | 200 |       40.87µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:18 | 200 |      40.045µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:18 | 200 |      37.506µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:29:19.051Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"f73765e8-d724-49ef-ad06-2fa528dd6d4e","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:29:19.051Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f73765e8-d724-49ef-ad06-2fa528dd6d4e","keyPrefix":"sk-oai-olj0otV0eksj...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:29:19 | 201 |    2.180118ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:29:23 | 200 |      38.888µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:28 | 200 |      49.716µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:28 | 200 |      38.458µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:33 | 200 |      38.636µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:35 | 200 |     933.834µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:29:35 | 403 |      97.156µs |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:29:38 | 200 |      38.797µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:38 | 200 |      37.409µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:43 | 200 |      42.734µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:48 | 200 |      46.466µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:48 | 200 |      39.478µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:52 | 403 |     116.593µs |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:29:53 | 200 |      38.297µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:58 | 200 |      36.839µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:29:58 | 200 |      38.468µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:30:01.326Z","caller":"api_keys/service.go:305","message":"Ephemeral key cleanup completed","deletedCount":0}
[GIN] 2026/05/29 - 17:30:01 | 200 |     911.989µs |     10.133.0.36 | POST     "/internal/v1/api-keys/cleanup"
[GIN] 2026/05/29 - 17:30:03 | 200 |      42.302µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:08 | 200 |      43.896µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:08 | 200 |      43.088µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:09 | 403 |     113.432µs |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:30:13 | 200 |      41.242µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:18 | 200 |      38.763µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:18 | 200 |      43.242µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:23 | 200 |      39.634µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:27 | 403 |     130.495µs |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:30:28 | 200 |      41.407µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:28 | 200 |      40.266µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:33 | 200 |      46.726µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:38 | 200 |      47.183µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:38 | 200 |      30.799µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:30:41.355Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-api-key-ignores-header-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"0d38c827-9199-4feb-97b6-5e383e58b591","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:30:41.355Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"0d38c827-9199-4feb-97b6-5e383e58b591","keyPrefix":"sk-oai-1F16P0QOa1sU...","username":"system:serviceaccount:default:e2e-api-key-ignores-header-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:30:41 | 201 |    2.266932ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:30:43 | 200 |       34.52µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:48 | 200 |      36.182µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:48 | 200 |      52.792µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:49 | 200 |     672.074µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:30:49 | 200 |     384.883µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:30:49 | 200 |     181.512µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:30:49 | 200 |   17.860083ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:30:53 | 200 |      52.271µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:58 | 200 |      34.359µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:30:58 | 200 |      34.442µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:03 | 200 |      32.727µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:31:03.703Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"24df3993-d20f-4a2b-9dca-934fc63c28f3","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:31:03.703Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"24df3993-d20f-4a2b-9dca-934fc63c28f3","keyPrefix":"sk-oai-RjDvpsp5dtho...","username":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:31:03 | 201 |    3.241793ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:31:03.740Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"ffe086b9-e592-4bc0-8c2a-ea117b20d213","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:31:03.740Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"ffe086b9-e592-4bc0-8c2a-ea117b20d213","keyPrefix":"sk-oai-1boxINCXtA5U...","username":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:31:03 | 201 |    1.737218ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:31:08 | 200 |      33.011µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:08 | 200 |      35.319µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:11 | 200 |      875.76µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:31:11 | 200 |     479.006µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:31:11 | 200 |     183.631µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:11 | 200 |    8.992222ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:31:11 | 200 |     456.852µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:31:11 | 200 |     425.664µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:31:11 | 200 |     133.258µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:11 | 200 |   10.537349ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:31:13 | 200 |      50.137µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:18 | 200 |      32.989µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:18 | 200 |      32.904µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:23 | 200 |      32.214µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:28 | 200 |      32.732µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:28 | 200 |      33.217µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:30 | 200 |     161.927µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:30 | 200 |     143.325µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:30 | 200 |     144.385µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:30 | 200 |   35.259104ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:31:33 | 200 |      32.877µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:38 | 200 |      34.236µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:38 | 200 |      34.577µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:43 | 200 |      33.546µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:48 | 200 |      34.664µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:48 | 200 |      34.404µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:49 | 200 |     184.299µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:49 | 200 |   12.005483ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:31:49 | 200 |     572.247µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:31:49 | 200 |   12.802215ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:31:53 | 200 |      38.111µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:58 | 200 |      39.675µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:31:58 | 200 |       57.27µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:03 | 200 |      44.517µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:08 | 200 |      39.157µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:08 | 200 |      42.465µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:13 | 200 |      38.573µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:32:15.168Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"731fa05a-1101-4275-b1ad-97b4c36a7b96","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:32:15.168Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"731fa05a-1101-4275-b1ad-97b4c36a7b96","keyPrefix":"sk-oai-qb5pFD0Uhy8c...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:32:15 | 201 |    1.943288ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:32:15 | 200 |     426.106µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:32:15 | 200 |     157.009µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:32:15 | 200 |     395.856µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:32:15 | 200 |     158.983µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:32:15 | 200 |     354.235µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:32:15 | 200 |     147.953µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:32:15 | 200 |     347.301µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:32:15 | 200 |      291.13µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:32:15 | 200 |     128.474µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:32:15 | 200 |    9.460191ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:32:18 | 200 |      39.185µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:18 | 200 |      40.051µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:23 | 200 |      39.822µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:28 | 200 |      40.706µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:28 | 200 |      39.309µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:32:33.149Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2e7eb300-f901-46d5-b1cd-d67e03dfac8c","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:32:33.149Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"2e7eb300-f901-46d5-b1cd-d67e03dfac8c","keyPrefix":"sk-oai-G7vo91XEbveV...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/29 - 17:32:33 | 201 |    1.388142ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:32:33 | 200 |      39.734µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:33 | 200 |     473.033µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:32:33 | 200 |     154.019µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:32:38 | 200 |      45.261µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:38 | 200 |      37.457µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:43 | 200 |      39.634µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:48 | 200 |      38.289µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:48 | 200 |       37.83µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:53 | 200 |      48.227µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:58 | 200 |      42.256µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:32:58 | 200 |      37.457µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:33:03 | 200 |      46.362µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:33:08 | 200 |      43.167µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:33:08 | 200 |       40.87µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:33:13 | 200 |       42.36µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:33:14.784Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"61d50fae-bcf7-4b29-b3fa-84dfc24857e9","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:14.784Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"61d50fae-bcf7-4b29-b3fa-84dfc24857e9","keyPrefix":"sk-oai-xWAbLU2bmiCH...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:14 | 201 |   10.160695ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:15.325Z","caller":"api_keys/service.go:150","message":"Created API key","user":"bob_sre","groups":["system:authenticated","Site-Reliability"],"id":"3beaf80f-3f24-441e-bba2-fd2a78c711df","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:15.325Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3beaf80f-3f24-441e-bba2-fd2a78c711df","keyPrefix":"sk-oai-zfBGogZ2TlKj...","username":"bob_sre","groups":["system:authenticated","Site-Reliability"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:15 | 201 |    2.044196ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:15.545Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"806b4951-be17-465f-b843-58bd9bcb60f3","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:15.545Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"806b4951-be17-465f-b843-58bd9bcb60f3","keyPrefix":"sk-oai-hQR6dN9pX6CG...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:15 | 201 |    1.647736ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:15 | 200 |    1.061399ms |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:15 | 200 |     344.974µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:15 | 200 |     116.541µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:33:15 | 200 |    9.829702ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:15 | 200 |     359.576µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:15 | 200 |     124.659µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
{"level":"info","timestamp":"2026-05-29T17:33:15.711Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"e36acbfa-03b1-4c44-9104-9cdd0d15e884","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:15.711Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e36acbfa-03b1-4c44-9104-9cdd0d15e884","keyPrefix":"sk-oai-YTh6lvDXVW76...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:15 | 201 |    1.575007ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:15.739Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"e36acbfa-03b1-4c44-9104-9cdd0d15e884"}
{"level":"info","timestamp":"2026-05-29T17:33:15.739Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"e36acbfa-03b1-4c44-9104-9cdd0d15e884","revokedBy":"alice_lead"}
[GIN] 2026/05/29 - 17:33:15 | 200 |    2.645274ms |      100.64.0.2 | DELETE   "/v1/api-keys/e36acbfa-03b1-4c44-9104-9cdd0d15e884" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:18 | 200 |       36.66µs |      10.132.0.2 | GET      "/health"
[GIN] 2026/05/29 - 17:33:18 | 200 |     953.585µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
{"level":"error","timestamp":"2026-05-29T17:33:18.770Z","caller":"token/handler.go:65","message":"Missing or empty username header","header":"X-MaaS-Username","stacktrace":"main.registerHandlers.(*Handler).ExtractUserInfo.func1\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:65\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/05/29 - 17:33:18 | 500 |      88.195µs |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:18 | 200 |      38.038µs |      10.132.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-29T17:33:19.064Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"ddbdc471-dc1b-4d4f-9c6c-cfe880a5caf8","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.064Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"ddbdc471-dc1b-4d4f-9c6c-cfe880a5caf8","keyPrefix":"sk-oai-mZoR5n0b5kN9...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |      1.6618ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.092Z","caller":"api_keys/service.go:150","message":"Created API key","user":"bob_sre","groups":["system:authenticated","Site-Reliability"],"id":"f5cbabe6-dfab-40fb-b7c8-0dfc40d61c8f","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.092Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f5cbabe6-dfab-40fb-b7c8-0dfc40d61c8f","keyPrefix":"sk-oai-EWPN5NmwLzpe...","username":"bob_sre","groups":["system:authenticated","Site-Reliability"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |    1.700047ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.185Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"9b594f50-15b6-468d-80d5-cf786a7ec1c5","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.185Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"9b594f50-15b6-468d-80d5-cf786a7ec1c5","keyPrefix":"sk-oai-1NbLI2xePDzP...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |    1.706759ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.216Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"9b594f50-15b6-468d-80d5-cf786a7ec1c5"}
{"level":"info","timestamp":"2026-05-29T17:33:19.216Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"9b594f50-15b6-468d-80d5-cf786a7ec1c5","revokedBy":"alice_lead"}
[GIN] 2026/05/29 - 17:33:19 | 200 |    2.126211ms |      100.64.0.2 | DELETE   "/v1/api-keys/9b594f50-15b6-468d-80d5-cf786a7ec1c5" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:19 | 404 |     579.888µs |      100.64.0.2 | DELETE   "/v1/api-keys/9b594f50-15b6-468d-80d5-cf786a7ec1c5" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.339Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"e37427c4-7d70-47e6-bcf7-c14dafcecf74","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.339Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e37427c4-7d70-47e6-bcf7-c14dafcecf74","keyPrefix":"sk-oai-KPlGGUtQI0XE...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |    1.638799ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:19 | 200 |     480.103µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     422.199µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     117.451µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:33:19 | 200 |    9.097446ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.471Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"924fb16d-1c65-4eae-9725-fc0dc8b4cd57","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.471Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"924fb16d-1c65-4eae-9725-fc0dc8b4cd57","keyPrefix":"sk-oai-Oh1eLkU6YzRj...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |    1.614841ms |      100.64.0.4 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:19 | 200 |     428.459µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 403 |     117.494µs |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:19 | 200 |     392.551µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     294.664µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     145.512µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:33:19 | 200 |    8.898337ms |      100.64.0.3 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.632Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"91cb2f47-c3cb-4038-8f09-0b1528701374","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.632Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"91cb2f47-c3cb-4038-8f09-0b1528701374","keyPrefix":"sk-oai-6rGMQYQIRpYJ...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |    1.589147ms |      100.64.0.3 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:19 | 200 |     365.232µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     330.803µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     119.424µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:33:19 | 200 |    9.507686ms |      100.64.0.2 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:19 | 200 |     292.066µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     309.019µs |     10.133.0.34 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/29 - 17:33:19 | 200 |     170.972µs |     10.133.0.34 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/29 - 17:33:19 | 200 |    8.863267ms |      100.64.0.4 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-05-29T17:33:19.793Z","caller":"api_keys/service.go:150","message":"Created API key","user":"bob_sre","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"32aaf843-2e13-419b-b87f-a83a5c683d41","ephemeral":false}
{"level":"info","timestamp":"2026-05-29T17:33:19.793Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"32aaf843-2e13-419b-b87f-a83a5c683d41","keyPrefix":"sk-oai-1elm50cunSbi...","username":"bob_sre","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false}
[GIN] 2026/05/29 - 17:33:19 | 201 |     1.63457ms |      100.64.0.2 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/05/29 - 17:33:23 | 200 |      40.094µs |      10.132.0.2 | GET      "/health"