[GIN] 2026/06/15 - 03:41:12 | 200 | 40.042µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:13 | 200 | 42.004µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:41:16.653Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"d94a1f06-7fec-455d-ab26-f56e1e66b88d","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:41:16.653Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"d94a1f06-7fec-455d-ab26-f56e1e66b88d","keyPrefix":"sk-oai-l1a21wMj0O9j...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:41:16 | 201 | 11.844832ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:41:16.691Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"59ff79b8-0b76-4ef1-901e-7ed469e62ab0","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:41:16.691Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"59ff79b8-0b76-4ef1-901e-7ed469e62ab0","keyPrefix":"sk-oai-y0aIPRboQ6ax...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:41:16 | 201 | 1.987896ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:41:16 | 200 | 1.261969ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:41:16 | 200 | 171.035µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:41:16 | 200 | 551.24µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:41:16 | 200 | 130.328µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:41:18 | 200 | 39.237µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:22 | 200 | 46.681µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:23 | 200 | 48.97µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:28 | 200 | 45.116µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:32 | 200 | 39.771µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:33 | 200 | 48.927µs | 10.133.0.2 | GET "/health" {"level":"warn","timestamp":"2026-06-15T03:41:34.382Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-user","requestedSubscription":"e2e-test-other-subscription","error":"access denied to requested subscription"} {"level":"error","timestamp":"2026-06-15T03:41:34.382Z","caller":"api_keys/handler.go:205","message":"Failed to create API key","error":"access denied to requested subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:205\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:136\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/06/15 - 03:41:34 | 400 | 261.638µs | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:41:38 | 200 | 39.432µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:42 | 200 | 42.549µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:43 | 200 | 49.32µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:48 | 200 | 35.204µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:41:51.748Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"646cdb65-afc9-4b57-8c33-54912eee025f","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:41:51.748Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"646cdb65-afc9-4b57-8c33-54912eee025f","keyPrefix":"sk-oai-HVUiIbazEvp1...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:41:51 | 201 | 2.067227ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:41:51 | 200 | 588.731µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:41:51 | 200 | 154.717µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:41:52 | 200 | 41.662µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:53 | 200 | 38.114µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:41:58 | 200 | 39.054µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:42:00.839Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-auth-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"3eebfdb9-8a93-4d9a-86d9-2cb741fb36b7","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:42:00.839Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"3eebfdb9-8a93-4d9a-86d9-2cb741fb36b7","keyPrefix":"sk-oai-1djwX1BShGmr...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-auth-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:42:00 | 201 | 2.453773ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:42:02 | 200 | 42.768µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:03 | 200 | 48.506µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:08 | 200 | 37.48µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:09 | 200 | 1.103015ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:42:09 | 200 | 361.023µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:42:12 | 200 | 39.453µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:13 | 200 | 37.317µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:18 | 200 | 38.129µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:42:18.295Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"4bd9c7f1-5c48-4de3-aef7-c1cfbc632994","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:42:18.295Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"4bd9c7f1-5c48-4de3-aef7-c1cfbc632994","keyPrefix":"sk-oai-Okjuv0NUkMwZ...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:42:18 | 201 | 3.009858ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:42:18 | 200 | 653.559µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:42:18 | 200 | 151.531µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:42:22 | 200 | 40.334µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:23 | 200 | 53.783µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:28 | 200 | 40.316µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:32 | 200 | 39.007µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:33 | 200 | 47.894µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:38 | 200 | 40.127µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:42 | 200 | 39.036µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:43 | 200 | 51.913µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:48 | 200 | 40.937µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:52 | 200 | 49.247µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:53 | 200 | 56.902µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:42:58 | 200 | 40.824µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:02 | 200 | 41.291µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:03 | 200 | 53.119µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:08 | 200 | 40.975µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:12 | 200 | 44.573µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:13 | 200 | 51.018µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:18 | 200 | 46.967µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:22 | 200 | 40.067µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:23 | 200 | 52.138µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:28 | 200 | 40.316µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:32 | 200 | 42.804µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:33 | 200 | 48.934µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:38 | 200 | 39.513µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:42 | 200 | 38.337µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:43 | 200 | 57.278µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:48 | 200 | 42.714µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:52 | 200 | 38.404µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:53 | 200 | 66.401µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:43:58 | 200 | 39.448µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:02 | 200 | 42.839µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:03 | 200 | 46.707µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:08 | 200 | 41.375µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:44:12.558Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"1bf2c3f9-4a8d-414a-8fce-e6168e4a85d7","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:44:12.558Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"1bf2c3f9-4a8d-414a-8fce-e6168e4a85d7","keyPrefix":"sk-oai-1Gt4sqao8GiX...","username":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:44:12 | 201 | 2.10812ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:44:12 | 200 | 620.377µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:44:12 | 200 | 181.122µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:44:12 | 200 | 35.333µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:13 | 200 | 61.006µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:18 | 200 | 37.833µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:22 | 200 | 42.584µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:23 | 200 | 46.899µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:28 | 200 | 37.268µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:32 | 200 | 47.213µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:33 | 200 | 52.24µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:33 | 200 | 896.999µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:44:33 | 200 | 270.239µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:44:34 | 404 | 1.170111ms | 52.71.36.72 | DELETE "/v1/api-keys/e2e-trlp-test-key" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:44:38 | 200 | 40.494µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:42 | 200 | 39.958µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:43 | 200 | 66.598µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:48 | 200 | 40.33µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:52 | 200 | 38.986µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:53 | 200 | 40.968µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:44:58 | 200 | 44.647µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:02 | 200 | 38.734µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:03 | 200 | 41.7µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:45:03.920Z","caller":"api_keys/service.go:307","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/06/15 - 03:45:03 | 200 | 1.088675ms | 10.132.0.39 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/06/15 - 03:45:08 | 200 | 244.278µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:12 | 200 | 44.515µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:13 | 200 | 46.053µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:18 | 200 | 61.662µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:45:22.579Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"4ad8da29-d1bb-49d6-8dc4-b5b1302a905c","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:45:22.579Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"4ad8da29-d1bb-49d6-8dc4-b5b1302a905c","keyPrefix":"sk-oai-1Uj5N31K256p...","username":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:45:22 | 201 | 2.078588ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:45:22 | 200 | 526.232µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:45:22 | 200 | 252.796µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:45:22 | 200 | 28.225µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:23 | 200 | 47.865µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:28 | 200 | 37.42µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:32 | 200 | 48.033µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:33 | 200 | 39.071µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:38 | 200 | 56.881µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:45:41.836Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"9a2c46e8-f225-4bdd-a282-1dd251ed48d4","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:45:41.836Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"9a2c46e8-f225-4bdd-a282-1dd251ed48d4","keyPrefix":"sk-oai-ZNWJvmp80PiT...","username":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:45:41 | 201 | 2.088883ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:45:41 | 200 | 504.517µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:45:41 | 200 | 127.308µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:45:42 | 200 | 631.597µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:45:42 | 200 | 222.966µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:45:42 | 200 | 39.969µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:43 | 200 | 46.533µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:48 | 200 | 44.56µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:52 | 200 | 42.647µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:53 | 200 | 43.954µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:45:58 | 200 | 54.067µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:46:01.343Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"ae103df3-8c46-4ed9-8eee-211b75167f91","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:46:01.343Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"ae103df3-8c46-4ed9-8eee-211b75167f91","keyPrefix":"sk-oai-1RmFE5t4jzqq...","username":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:46:01 | 201 | 2.085284ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:46:01 | 200 | 641.557µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:46:01 | 200 | 668.103µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:46:01 | 200 | 354.652µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:46:01 | 200 | 19.998233ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:46:02 | 200 | 41.826µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:03 | 200 | 43.468µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:08 | 200 | 39.309µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:12 | 200 | 40.681µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:13 | 200 | 50.84µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:18 | 200 | 52.797µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:20 | 200 | 179.76µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:46:20 | 200 | 166.045µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:46:20 | 200 | 34.348486ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:46:22 | 200 | 45.307µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:23 | 200 | 46.978µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:28 | 200 | 44.263µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:46:32.561Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2c53b9eb-43f4-475c-afaf-f24571c9204f","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:46:32.561Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"2c53b9eb-43f4-475c-afaf-f24571c9204f","keyPrefix":"sk-oai-OwzZClSPK899...","username":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:46:32 | 201 | 10.699209ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:46:32 | 200 | 35.791µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:33 | 200 | 47.808µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:38 | 200 | 47.586µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:42 | 200 | 39.178µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:43 | 200 | 48.966µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:48 | 200 | 39.739µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:52 | 200 | 45.083µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:53 | 200 | 46.94µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:46:58 | 200 | 43.3µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:02 | 200 | 1.691873ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:02 | 200 | 166.743µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:47:02 | 200 | 652.615µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:02 | 200 | 30.318µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:02 | 200 | 495.782µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:02 | 200 | 115.208µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:47:02 | 200 | 13.090977ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:03 | 200 | 44.401µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:08 | 200 | 40.85µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:12 | 200 | 44.716µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:13 | 200 | 47.955µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:18 | 200 | 39.372µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:19 | 200 | 170.175µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:47:19 | 200 | 17.956603ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:22 | 200 | 54.733µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:23 | 200 | 45.492µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:28 | 200 | 40.294µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:47:28.413Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1865db18-4df6-48fe-b2dd-d02d18448c2e","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:47:28.413Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"1865db18-4df6-48fe-b2dd-d02d18448c2e","keyPrefix":"sk-oai-80qmrSt0Go9V...","username":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:47:28 | 201 | 2.130873ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:32 | 200 | 46.234µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:33 | 200 | 41.571µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:36 | 200 | 1.059947ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:36 | 200 | 524.047µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:36 | 200 | 120.242µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:47:36 | 200 | 11.272222ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:47:36.992Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fdfff7d5-7929-4f85-9761-b8372fca57c5","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:47:36.992Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"fdfff7d5-7929-4f85-9761-b8372fca57c5","keyPrefix":"sk-oai-1Ou7kCb5n5ZT...","username":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:47:36 | 201 | 1.806759ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:38 | 200 | 45.048µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:42 | 200 | 57.343µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:43 | 200 | 49.354µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:45 | 200 | 961.942µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:45 | 200 | 388.294µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:45 | 200 | 127.824µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:47:45 | 200 | 6.677154ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:45 | 200 | 495.133µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:45 | 200 | 385.821µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:47:45 | 200 | 99.258µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:47:45 | 200 | 6.456376ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:47:46.002Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"53985801-a56e-458d-af4b-f1ea2fa37430","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:47:46.002Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"53985801-a56e-458d-af4b-f1ea2fa37430","keyPrefix":"sk-oai-fPlnowMHQ7k4...","username":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:47:46 | 201 | 1.81144ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:48 | 200 | 38.004µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:52 | 200 | 54.147µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:53 | 200 | 46.116µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:47:54 | 200 | 721.712µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" {"level":"error","timestamp":"2026-06-15T03:47:54.040Z","caller":"token/handler.go:65","message":"Missing or empty username header","header":"X-MaaS-Username","stacktrace":"main.registerHandlers.(*Handler).ExtractUserInfo.func1\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:65\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/06/15 - 03:47:54 | 500 | 96.929µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:47:58 | 200 | 39.639µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:02 | 200 | 40.765µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:03 | 200 | 48.892µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:48:03.272Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"c724a972-1480-4d26-86c0-2d36ddfa4474","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:48:03.272Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"c724a972-1480-4d26-86c0-2d36ddfa4474","keyPrefix":"sk-oai-E93lAKPWnpta...","username":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:48:03 | 201 | 2.012556ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:48:08 | 200 | 43.297µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:11 | 200 | 765.135µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:48:11 | 200 | 487.067µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:48:11 | 200 | 247.314µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:11 | 200 | 385.613µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:48:11 | 200 | 255.725µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:11 | 200 | 18.490725ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:48:12 | 200 | 45.067µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:13 | 200 | 49.789µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:18 | 200 | 52.967µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:22 | 200 | 45.254µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:23 | 200 | 57.438µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:48:24.782Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"60d667ff-57db-4df4-969f-f8d990927a8a","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:48:24.782Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"60d667ff-57db-4df4-969f-f8d990927a8a","keyPrefix":"sk-oai-1DZzJjjq4Jsd...","username":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:48:24 | 201 | 3.923872ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:48:28 | 200 | 44.118µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:32 | 200 | 40.475µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:32 | 200 | 940.583µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:48:32 | 200 | 565.69µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:48:32 | 200 | 251.241µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:32 | 200 | 10.134692ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:48:32 | 200 | 206.574µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:32 | 200 | 24.201824ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:48:33 | 200 | 47.69µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:38 | 200 | 40.063µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:42 | 200 | 45.545µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:43 | 200 | 48.097µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:44 | 200 | 192.79µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:44 | 200 | 150.764µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:44 | 200 | 148.052µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:48:44 | 200 | 55.141755ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:48:48 | 200 | 63.147µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:52 | 200 | 46.087µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:53 | 200 | 42.069µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:48:58 | 200 | 42.534µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:02 | 200 | 221.565µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:49:02 | 200 | 27.187005ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:02 | 200 | 37.763µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:03 | 200 | 50.584µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:08 | 200 | 53.846µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:12 | 200 | 40.784µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:13 | 200 | 39.911µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:49:13.405Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1881236c-1bd8-4019-97be-1fa8ad0ada9e","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:49:13.405Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"1881236c-1bd8-4019-97be-1fa8ad0ada9e","keyPrefix":"sk-oai-AHvXKQaFUjSU...","username":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:49:13 | 201 | 2.417143ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:18 | 200 | 41.292µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:21 | 200 | 911.365µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:21 | 200 | 130.359µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:49:22.038Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"826248e4-41dc-4215-9afd-49e95987e3fd","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:49:22.038Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"826248e4-41dc-4215-9afd-49e95987e3fd","keyPrefix":"sk-oai-1AZmWLC4UamY...","username":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:49:22 | 201 | 1.841694ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:22 | 200 | 41.738µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:23 | 200 | 50.808µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:28 | 200 | 40.296µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:30 | 200 | 1.358729ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:30 | 200 | 444.43µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:30 | 200 | 124.72µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:49:30 | 200 | 12.894605ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:49:30.451Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"83645678-5077-4f40-9556-1ae1e86a2c7b","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:49:30.451Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"83645678-5077-4f40-9556-1ae1e86a2c7b","keyPrefix":"sk-oai-1MVQBM6zgFsm...","username":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:49:30 | 201 | 1.903794ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:32 | 200 | 41.637µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:33 | 200 | 41.154µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:38 | 200 | 44.618µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:38 | 200 | 998.235µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:38 | 200 | 471.604µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:38 | 200 | 154.822µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:49:38 | 200 | 10.685819ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:49:39.281Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"8ef205c9-3685-4b75-b9f5-53279361686e","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:49:39.281Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"8ef205c9-3685-4b75-b9f5-53279361686e","keyPrefix":"sk-oai-1KlFOfagHBu4...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:49:39 | 201 | 1.901857ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:42 | 200 | 39.455µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:43 | 200 | 47.502µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:47 | 200 | 816.463µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:47 | 200 | 415.459µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:49:47 | 200 | 185.225µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:49:47 | 200 | 11.462083ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:48 | 200 | 45.532µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:52 | 200 | 39.502µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:49:53 | 200 | 50.442µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:49:56.502Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"c5de3a37-a366-4e57-aa6e-a923d61ec326","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:49:56.502Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"c5de3a37-a366-4e57-aa6e-a923d61ec326","keyPrefix":"sk-oai-wrgLDxNhbjtN...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:49:56 | 201 | 2.735828ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:49:58 | 200 | 50.434µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:02 | 200 | 40.555µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:03 | 200 | 42.828µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:08 | 200 | 42.815µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:12 | 200 | 39.413µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:12 | 200 | 853.776µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:50:12 | 403 | 107.214µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:50:13 | 200 | 48.537µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:18 | 200 | 44.754µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:22 | 200 | 40.094µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:23 | 200 | 47.881µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:28 | 200 | 42.634µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:29 | 403 | 146.856µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:50:32 | 200 | 43.176µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:33 | 200 | 45.301µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:38 | 200 | 45.618µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:42 | 200 | 40.35µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:43 | 200 | 47.281µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:47 | 403 | 151.227µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:50:48 | 200 | 39.101µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:52 | 200 | 39.141µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:53 | 200 | 48.759µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:50:58 | 200 | 50.649µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:02 | 200 | 40.938µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:03 | 200 | 44.508µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:04 | 403 | 148.975µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:08 | 200 | 40.104µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:12 | 200 | 42.965µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:13 | 200 | 40.445µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:51:16.355Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-api-key-ignores-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"0c4d07cd-1303-4240-9e46-6b99f7f1070f","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:51:16.355Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"0c4d07cd-1303-4240-9e46-6b99f7f1070f","keyPrefix":"sk-oai-17OnBgfRdK3x...","username":"system:serviceaccount:default:e2e-api-key-ignores-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:51:16 | 201 | 2.126639ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:18 | 200 | 38.902µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:22 | 200 | 39.835µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:23 | 200 | 43.387µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:24 | 200 | 813.734µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:51:24 | 200 | 424.773µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:51:24 | 200 | 138.228µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:51:24 | 200 | 18.294186ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:28 | 200 | 40.857µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:32 | 200 | 39.307µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:33 | 200 | 50.022µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:51:36.487Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"bfb6bb57-9d99-46a5-a17a-3c9792a433af","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:51:36.487Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"bfb6bb57-9d99-46a5-a17a-3c9792a433af","keyPrefix":"sk-oai-17RYtSvhnCkL...","username":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:51:36 | 201 | 2.171512ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:51:36.523Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"58c7d922-2aa8-4b10-8418-62d048c8e528","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:51:36.523Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"58c7d922-2aa8-4b10-8418-62d048c8e528","keyPrefix":"sk-oai-1daKQFveoGvg...","username":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:51:36 | 201 | 1.801069ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:38 | 200 | 42.14µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:42 | 200 | 40.719µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:43 | 200 | 43.184µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:44 | 200 | 885.31µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:51:44 | 200 | 440.352µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:51:44 | 200 | 136.426µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:51:44 | 200 | 11.7941ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:44 | 200 | 540.552µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:51:44 | 200 | 389.32µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:51:44 | 200 | 120.738µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:51:44 | 200 | 12.480468ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:48 | 200 | 52.614µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:52 | 200 | 38.373µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:53 | 200 | 47.938µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:51:56 | 200 | 233.658µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:51:56 | 200 | 394.322µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:51:56 | 200 | 355.571µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:51:56 | 200 | 54.718311ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:51:58 | 200 | 38.761µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:02 | 200 | 57.087µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:03 | 200 | 41.795µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:08 | 200 | 38.675µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:09 | 200 | 204.798µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:09 | 200 | 20.761649ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:52:09 | 200 | 318.601µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:09 | 200 | 18.772285ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:52:12 | 200 | 41.791µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:13 | 200 | 43.63µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:18 | 200 | 41.882µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:22 | 200 | 42.946µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:23 | 200 | 35.011µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:28 | 200 | 40.508µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:32 | 200 | 64.185µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:33 | 200 | 55.13µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:52:35.176Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-central-models-exempt-sa-bfe1a3","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"059ab8b6-ab31-46ac-93b9-6803daca3391","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:52:35.176Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"059ab8b6-ab31-46ac-93b9-6803daca3391","keyPrefix":"sk-oai-1WXEoMniT52p...","username":"system:serviceaccount:models-as-a-service:e2e-central-models-exempt-sa-bfe1a3","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:52:35 | 201 | 2.549772ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:52:35 | 200 | 535.052µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:35 | 200 | 240.974µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:35 | 200 | 631.596µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:35 | 200 | 247.21µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:35 | 200 | 551.682µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:35 | 200 | 169.339µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:35 | 200 | 461.401µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:35 | 200 | 193.052µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:35 | 200 | 486.653µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:35 | 200 | 419.057µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:35 | 200 | 170.294µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:35 | 200 | 10.989892ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:52:38 | 200 | 41.68µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:42 | 200 | 38.461µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:43 | 200 | 52.837µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:48 | 200 | 39.523µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:52:48.999Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"928e479b-76a3-4c39-a6ce-044284ec51dd","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:52:48.999Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"928e479b-76a3-4c39-a6ce-044284ec51dd","keyPrefix":"sk-oai-IxyLz1H7jvWd...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/06/15 - 03:52:48 | 201 | 3.631187ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:52:49 | 200 | 555.015µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:52:49 | 200 | 154.128µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:52:52 | 200 | 40.799µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:53 | 200 | 46.328µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:52:58 | 200 | 40.548µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:02 | 200 | 38.447µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:03 | 200 | 64.554µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:08 | 200 | 53.675µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:12 | 200 | 40.023µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:13 | 200 | 39.168µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:18 | 200 | 39.498µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:22 | 200 | 41.225µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:23 | 200 | 54.941µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:28 | 200 | 38.351µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:32 | 200 | 38.778µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:33 | 200 | 47.501µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:38 | 200 | 59.739µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:42 | 200 | 41.191µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:43 | 200 | 48.991µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:48 | 200 | 41.748µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:52 | 200 | 41.021µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:53 | 200 | 46.675µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-06-15T03:53:54.856Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"004007b6-bd21-418f-8e6d-922e6d690fc9","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:54.856Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"004007b6-bd21-418f-8e6d-922e6d690fc9","keyPrefix":"sk-oai-nVxzorxY3RZo...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:54 | 201 | 11.450365ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:55.406Z","caller":"api_keys/service.go:150","message":"Created API key","user":"bob_sre","groups":["system:authenticated","Site-Reliability"],"id":"71973630-9931-4bb5-9f9a-6ca3eba8c2aa","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:55.406Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"71973630-9931-4bb5-9f9a-6ca3eba8c2aa","keyPrefix":"sk-oai-Zsr0ZDPS6COO...","username":"bob_sre","groups":["system:authenticated","Site-Reliability"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:55 | 201 | 2.110355ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:55.627Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"1dc444b8-2614-4c25-91a6-b0662395541d","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:55.627Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"1dc444b8-2614-4c25-91a6-b0662395541d","keyPrefix":"sk-oai-XxLc44ywOnog...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:55 | 201 | 1.815964ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:55 | 200 | 1.377403ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:55 | 200 | 651.942µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:55 | 200 | 391.116µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:53:55 | 200 | 11.689245ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:55 | 200 | 876.96µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:55 | 200 | 376.282µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-06-15T03:53:55.812Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"1750e9f0-db7b-41b8-bd64-3f4b167b479b","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:55.812Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"1750e9f0-db7b-41b8-bd64-3f4b167b479b","keyPrefix":"sk-oai-QGnRPAP3jto8...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:55 | 201 | 1.829188ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:55.842Z","caller":"api_keys/store_postgres.go:474","message":"Revoked API key","id":"1750e9f0-db7b-41b8-bd64-3f4b167b479b"} {"level":"info","timestamp":"2026-06-15T03:53:55.842Z","caller":"api_keys/handler.go:339","message":"Revoked API key","keyId":"1750e9f0-db7b-41b8-bd64-3f4b167b479b","revokedBy":"alice_lead"} [GIN] 2026/06/15 - 03:53:55 | 200 | 2.924533ms | 52.71.36.72 | DELETE "/v1/api-keys/1750e9f0-db7b-41b8-bd64-3f4b167b479b" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:58 | 200 | 45.769µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:53:58 | 200 | 1.10085ms | 10.132.0.38 | POST "/internal/v1/api-keys/validate" {"level":"info","timestamp":"2026-06-15T03:53:59.169Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"b99edd2f-6e49-4db6-9bd2-ab97f012357e","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.169Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"b99edd2f-6e49-4db6-9bd2-ab97f012357e","keyPrefix":"sk-oai-mgykQtgtLOwF...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.88249ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.202Z","caller":"api_keys/service.go:150","message":"Created API key","user":"bob_sre","groups":["system:authenticated","Site-Reliability"],"id":"2fde642a-ec29-44e7-a7e7-c3b20f511f98","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.202Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"2fde642a-ec29-44e7-a7e7-c3b20f511f98","keyPrefix":"sk-oai-1SaU5XOjWPDs...","username":"bob_sre","groups":["system:authenticated","Site-Reliability"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.723463ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.297Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"2e840f4f-bb28-4459-91f8-e045f4a845dc","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.297Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"2e840f4f-bb28-4459-91f8-e045f4a845dc","keyPrefix":"sk-oai-VVE9UREL88hX...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.758734ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.328Z","caller":"api_keys/store_postgres.go:474","message":"Revoked API key","id":"2e840f4f-bb28-4459-91f8-e045f4a845dc"} {"level":"info","timestamp":"2026-06-15T03:53:59.328Z","caller":"api_keys/handler.go:339","message":"Revoked API key","keyId":"2e840f4f-bb28-4459-91f8-e045f4a845dc","revokedBy":"alice_lead"} [GIN] 2026/06/15 - 03:53:59 | 200 | 2.290903ms | 52.71.36.72 | DELETE "/v1/api-keys/2e840f4f-bb28-4459-91f8-e045f4a845dc" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:59 | 404 | 695.058µs | 52.71.36.72 | DELETE "/v1/api-keys/2e840f4f-bb28-4459-91f8-e045f4a845dc" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.454Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"cb9ee9c9-3115-4888-895f-64e3cbecf475","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.455Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"cb9ee9c9-3115-4888-895f-64e3cbecf475","keyPrefix":"sk-oai-12kwCFSN7BHq...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.678961ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:59 | 200 | 657.264µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 542.162µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 212.72µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:53:59 | 200 | 10.802747ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.583Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"4a957bdf-56ce-4b85-9f68-64dca45c6465","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.583Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"4a957bdf-56ce-4b85-9f68-64dca45c6465","keyPrefix":"sk-oai-195XYG5t3x8c...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.666292ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:59 | 200 | 567.588µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 403 | 77.299µs | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:59 | 200 | 534.897µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 469.259µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 249.327µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:53:59 | 200 | 10.097971ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.744Z","caller":"api_keys/service.go:150","message":"Created API key","user":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"50517fc0-f819-40a6-928e-149f7a5f6d81","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.744Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"50517fc0-f819-40a6-928e-149f7a5f6d81","keyPrefix":"sk-oai-1ZMpZDQxh3RO...","username":"alice_lead","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.845619ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:59 | 200 | 566.319µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 416.65µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 259.558µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:53:59 | 200 | 9.997742ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:53:59 | 200 | 538.035µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 443.294µs | 10.132.0.38 | POST "/internal/v1/api-keys/validate" [GIN] 2026/06/15 - 03:53:59 | 200 | 219.68µs | 10.132.0.38 | POST "/internal/v1/subscriptions/select" [GIN] 2026/06/15 - 03:53:59 | 200 | 10.048779ms | 52.71.36.72 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-06-15T03:53:59.915Z","caller":"api_keys/service.go:150","message":"Created API key","user":"bob_sre","groups":["system:authenticated","Engineering","Project-Alpha"],"id":"4b2a55ea-fa29-4f96-99d0-a21c8504f6b1","ephemeral":false} {"level":"info","timestamp":"2026-06-15T03:53:59.915Z","caller":"api_keys/handler.go:238","message":"Created API key","keyId":"4b2a55ea-fa29-4f96-99d0-a21c8504f6b1","keyPrefix":"sk-oai-16Gl7LwOko0H...","username":"bob_sre","groups":["system:authenticated","Engineering","Project-Alpha"],"ephemeral":false} [GIN] 2026/06/15 - 03:53:59 | 201 | 1.815965ms | 52.71.36.72 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/06/15 - 03:54:02 | 200 | 42.461µs | 10.133.0.2 | GET "/health" [GIN] 2026/06/15 - 03:54:03 | 200 | 43.871µs | 10.133.0.2 | GET "/health"