{"level":"info","timestamp":"2026-05-21T01:05:47.051Z","caller":"cmd/main.go:65","message":"Loading database connection URL from secret..."}
{"level":"info","timestamp":"2026-05-21T01:05:47.064Z","caller":"cmd/main.go:174","message":"Connecting to PostgreSQL database..."}
{"level":"info","timestamp":"2026-05-21T01:05:47.064Z","caller":"cmd/main.go:103","message":"Metrics server starting","address":":9090"}
{"level":"info","timestamp":"2026-05-21T01:05:47.230Z","caller":"api_keys/db_driver.go:90","message":"Database schema applied","version":3}
{"level":"info","timestamp":"2026-05-21T01:05:47.230Z","caller":"api_keys/db_driver.go:57","message":"Connected to PostgreSQL database (schema applied)"}
{"level":"info","timestamp":"2026-05-21T01:05:47.334Z","caller":"cmd/main.go:195","message":"Resolved gateway internal host for access probes","host":"maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local"}
{"level":"info","timestamp":"2026-05-21T01:05:47.334Z","caller":"cmd/main.go:136","message":"Server starting","address":":8443","secure":true}
[GIN] 2026/05/21 - 01:05:53 | 200 |      63.447µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:05:58 | 200 |      33.138µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:03 | 200 |      33.105µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:08 | 200 |      39.949µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:13 | 200 |      35.987µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:18 | 200 |      31.473µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:23 | 200 |      33.742µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:25 | 200 |      32.996µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:28 | 200 |       33.18µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:33 | 200 |      53.913µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:35 | 200 |      42.354µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:38 | 200 |      43.729µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:43 | 200 |      33.466µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:45 | 200 |      42.767µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:48 | 200 |       37.57µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:53 | 200 |      33.061µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:55 | 200 |      33.127µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:06:58 | 200 |      35.113µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:03 | 200 |      31.531µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:05 | 200 |      35.897µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:08 | 200 |      32.816µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:13 | 200 |      34.757µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:15 | 200 |      32.001µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:18 | 200 |      32.658µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:23 | 200 |       44.13µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:25 | 200 |      32.285µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:28 | 200 |      31.695µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:32 | 200 |      21.198µs |      100.64.0.3 | GET      "/health"
[GIN] 2026/05/21 - 01:07:33 | 200 |      34.469µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:07:33.321Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e35379b2-68b1-476b-889d-b5c2f81280e3","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.321Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e35379b2-68b1-476b-889d-b5c2f81280e3","keyPrefix":"sk-oai-1XZ1bcx0jm07...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |   11.858805ms |      100.64.0.2 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:33 | 200 |    1.279332ms |      100.64.0.4 | GET      "/v1/api-keys/e35379b2-68b1-476b-889d-b5c2f81280e3"
{"level":"info","timestamp":"2026-05-21T01:07:33.396Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"6aaa3273-2c19-47c0-aa4e-7349ce76816d","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.396Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"6aaa3273-2c19-47c0-aa4e-7349ce76816d","keyPrefix":"sk-oai-86OmN9S23JCl...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.668327ms |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:33.432Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"33fc29d5-be10-4122-bb0e-3238468f46d9","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.432Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"33fc29d5-be10-4122-bb0e-3238468f46d9","keyPrefix":"sk-oai-12j4zaOrBeHF...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.572309ms |      100.64.0.3 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:33 | 200 |    4.162922ms |      100.64.0.2 | POST     "/v1/api-keys/search"
[GIN] 2026/05/21 - 01:07:33 | 200 |     556.701µs |      100.64.0.4 | POST     "/v1/api-keys/search"
{"level":"info","timestamp":"2026-05-21T01:07:33.541Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"62794f19-31a4-402f-990e-dac590df896e","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.541Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"62794f19-31a4-402f-990e-dac590df896e","keyPrefix":"sk-oai-1eNXiC5P7NgV...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.698115ms |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:33.571Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"62794f19-31a4-402f-990e-dac590df896e"}
{"level":"info","timestamp":"2026-05-21T01:07:33.571Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"62794f19-31a4-402f-990e-dac590df896e","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:33 | 200 |    2.565732ms |      100.64.0.2 | DELETE   "/v1/api-keys/62794f19-31a4-402f-990e-dac590df896e"
[GIN] 2026/05/21 - 01:07:33 | 200 |     425.679µs |      100.64.0.4 | GET      "/v1/api-keys/62794f19-31a4-402f-990e-dac590df896e"
{"level":"info","timestamp":"2026-05-21T01:07:33.656Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fe9217c1-0bd2-4933-baf1-2c66ab25fc1e","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.656Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"fe9217c1-0bd2-4933-baf1-2c66ab25fc1e","keyPrefix":"sk-oai-T0zmZbKXBCmA...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |     1.62373ms |      100.64.0.3 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:33 | 200 |     409.263µs |      100.64.0.2 | GET      "/v1/api-keys/fe9217c1-0bd2-4933-baf1-2c66ab25fc1e"
[GIN] 2026/05/21 - 01:07:33 | 200 |    3.632662ms |      100.64.0.4 | POST     "/v1/api-keys/search"
{"level":"info","timestamp":"2026-05-21T01:07:33.740Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"fe9217c1-0bd2-4933-baf1-2c66ab25fc1e"}
{"level":"info","timestamp":"2026-05-21T01:07:33.740Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"fe9217c1-0bd2-4933-baf1-2c66ab25fc1e","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"}
[GIN] 2026/05/21 - 01:07:33 | 200 |     2.10667ms |      100.64.0.3 | DELETE   "/v1/api-keys/fe9217c1-0bd2-4933-baf1-2c66ab25fc1e"
{"level":"info","timestamp":"2026-05-21T01:07:33.783Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"id":"e48a0c03-6952-4165-ac1a-3f38ac86c655","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.783Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e48a0c03-6952-4165-ac1a-3f38ac86c655","keyPrefix":"sk-oai-1KHWFYUGnYj2...","username":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.705386ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"warn","timestamp":"2026-05-21T01:07:33.813Z","caller":"api_keys/handler.go:124","message":"Unauthorized API key access attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"e48a0c03-6952-4165-ac1a-3f38ac86c655"}
[GIN] 2026/05/21 - 01:07:33 | 404 |     371.512µs |      100.64.0.2 | GET      "/v1/api-keys/e48a0c03-6952-4165-ac1a-3f38ac86c655"
{"level":"warn","timestamp":"2026-05-21T01:07:33.848Z","caller":"api_keys/handler.go:305","message":"Unauthorized API key revocation attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"e48a0c03-6952-4165-ac1a-3f38ac86c655"}
[GIN] 2026/05/21 - 01:07:33 | 404 |     376.924µs |      100.64.0.4 | DELETE   "/v1/api-keys/e48a0c03-6952-4165-ac1a-3f38ac86c655"
{"level":"info","timestamp":"2026-05-21T01:07:33.878Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"53bc213b-c391-41cc-8eab-63c7774220ba","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.878Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"53bc213b-c391-41cc-8eab-63c7774220ba","keyPrefix":"sk-oai-1TyjewN1xQmp...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.887639ms |      100.64.0.4 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:33.906Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d0a8c299-6150-4148-8a4e-0412c966bc24","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.906Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d0a8c299-6150-4148-8a4e-0412c966bc24","keyPrefix":"sk-oai-1HPFM7XrWgl7...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.600179ms |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:33.931Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"b819e50c-7c79-4585-8af2-aa72d2d09254","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:33.931Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b819e50c-7c79-4585-8af2-aa72d2d09254","keyPrefix":"sk-oai-nUwKbvx5z0fR...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:33 | 201 |    1.689547ms |      100.64.0.2 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:33 | 200 |      385.88µs |      100.64.0.4 | GET      "/v1/api-keys/53bc213b-c391-41cc-8eab-63c7774220ba"
{"level":"info","timestamp":"2026-05-21T01:07:33.979Z","caller":"api_keys/store_postgres.go:440","message":"Revoked all keys for user","count":6,"user":"system:serviceaccount:default:tester-regular-user"}
{"level":"info","timestamp":"2026-05-21T01:07:33.979Z","caller":"api_keys/handler.go:531","message":"Bulk revoked API keys","count":6,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:33 | 200 |    1.991714ms |      100.64.0.3 | POST     "/v1/api-keys/bulk-revoke"
[GIN] 2026/05/21 - 01:07:34 | 200 |     345.323µs |      100.64.0.3 | GET      "/v1/api-keys/53bc213b-c391-41cc-8eab-63c7774220ba"
[GIN] 2026/05/21 - 01:07:34 | 200 |     342.089µs |      100.64.0.2 | GET      "/v1/api-keys/d0a8c299-6150-4148-8a4e-0412c966bc24"
[GIN] 2026/05/21 - 01:07:34 | 200 |     360.512µs |      100.64.0.4 | GET      "/v1/api-keys/b819e50c-7c79-4585-8af2-aa72d2d09254"
{"level":"warn","timestamp":"2026-05-21T01:07:34.087Z","caller":"api_keys/handler.go:508","message":"Unauthorized bulk revoke attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","targetUser":"someotheruser"}
[GIN] 2026/05/21 - 01:07:34 | 403 |      97.803µs |      100.64.0.3 | POST     "/v1/api-keys/bulk-revoke"
{"level":"info","timestamp":"2026-05-21T01:07:34.115Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a25a8b9c-eecd-46c4-9f16-9d27eefa93ed","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.115Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"a25a8b9c-eecd-46c4-9f16-9d27eefa93ed","keyPrefix":"sk-oai-5C9OYrMk2pU0...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.629943ms |      100.64.0.2 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:34 | 200 |      388.23µs |      100.64.0.4 | GET      "/v1/api-keys/a25a8b9c-eecd-46c4-9f16-9d27eefa93ed"
{"level":"info","timestamp":"2026-05-21T01:07:34.165Z","caller":"api_keys/store_postgres.go:440","message":"Revoked all keys for user","count":1,"user":"system:serviceaccount:default:tester-regular-user"}
{"level":"info","timestamp":"2026-05-21T01:07:34.165Z","caller":"api_keys/handler.go:531","message":"Bulk revoked API keys","count":1,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"}
[GIN] 2026/05/21 - 01:07:34 | 200 |    1.618982ms |      100.64.0.3 | POST     "/v1/api-keys/bulk-revoke"
{"level":"info","timestamp":"2026-05-21T01:07:34.196Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a9a37ffe-74d3-48a3-8584-3d9c8ad8cead","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.196Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"a9a37ffe-74d3-48a3-8584-3d9c8ad8cead","keyPrefix":"sk-oai-qwlh90kB6gHM...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.611364ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:34.226Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f523836c-a0c3-49ce-b75b-0537acd48726","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.226Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f523836c-a0c3-49ce-b75b-0537acd48726","keyPrefix":"sk-oai-18h5jXpLgx5V...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.557467ms |      100.64.0.4 | POST     "/v1/api-keys"
{"level":"error","timestamp":"2026-05-21T01:07:34.253Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested expiration (4320h0m0s) exceeds maximum allowed (90 days): expiration exceeds maximum allowed","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/05/21 - 01:07:34 | 400 |     157.411µs |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:34.279Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"428e78ed-5bf8-4331-8349-253f797d1c74","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.280Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"428e78ed-5bf8-4331-8349-253f797d1c74","keyPrefix":"sk-oai-1RP9F0dm3Muc...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.597461ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:34.306Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d3a30388-0a97-4c6c-8f4e-4c2665367cc5","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.306Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d3a30388-0a97-4c6c-8f4e-4c2665367cc5","keyPrefix":"sk-oai-zPcRYXRJypMu...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.804999ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:34.334Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"c60a3169-d36a-4160-aa70-96ecc5b98730","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.334Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"c60a3169-d36a-4160-aa70-96ecc5b98730","keyPrefix":"sk-oai-1VjaOQjjcewz...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.695354ms |      100.64.0.4 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:34 | 200 |   11.451887ms |      100.64.0.3 | GET      "/v1/models"
{"level":"info","timestamp":"2026-05-21T01:07:34.518Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e47f2660-b044-4b5d-bd29-3ee1ef07c63b","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:34.518Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e47f2660-b044-4b5d-bd29-3ee1ef07c63b","keyPrefix":"sk-oai-rjMrB4Ctyyxa...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:34 | 201 |    1.781109ms |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:34.564Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"e47f2660-b044-4b5d-bd29-3ee1ef07c63b"}
{"level":"info","timestamp":"2026-05-21T01:07:34.564Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"e47f2660-b044-4b5d-bd29-3ee1ef07c63b","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:34 | 200 |    2.069844ms |      100.64.0.2 | DELETE   "/v1/api-keys/e47f2660-b044-4b5d-bd29-3ee1ef07c63b"
[GIN] 2026/05/21 - 01:07:35 | 200 |      45.504µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:07:36.638Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d12b6f49-976c-4058-b30b-7df31008e154","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:36.638Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d12b6f49-976c-4058-b30b-7df31008e154","keyPrefix":"sk-oai-rslKURx6IavF...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:36 | 201 |    1.927312ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:36.665Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"d12b6f49-976c-4058-b30b-7df31008e154"}
{"level":"info","timestamp":"2026-05-21T01:07:36.665Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"d12b6f49-976c-4058-b30b-7df31008e154","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:36 | 200 |    2.083758ms |      100.64.0.4 | DELETE   "/v1/api-keys/d12b6f49-976c-4058-b30b-7df31008e154"
[GIN] 2026/05/21 - 01:07:36 | 404 |     654.624µs |      100.64.0.2 | DELETE   "/v1/api-keys/d12b6f49-976c-4058-b30b-7df31008e154"
[GIN] 2026/05/21 - 01:07:36 | 404 |       351.1µs |      100.64.0.3 | DELETE   "/v1/api-keys/nonexistent-uuid-12345"
{"level":"info","timestamp":"2026-05-21T01:07:36.747Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3bd5a043-1e65-4564-98e3-1226fd586222","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:36.747Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3bd5a043-1e65-4564-98e3-1226fd586222","keyPrefix":"sk-oai-OdjPY5kGWeUJ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:36 | 201 |    1.608721ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:36.774Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"3bd5a043-1e65-4564-98e3-1226fd586222"}
{"level":"info","timestamp":"2026-05-21T01:07:36.774Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"3bd5a043-1e65-4564-98e3-1226fd586222","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:36 | 200 |    1.952935ms |      100.64.0.4 | DELETE   "/v1/api-keys/3bd5a043-1e65-4564-98e3-1226fd586222"
{"level":"info","timestamp":"2026-05-21T01:07:36.805Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1ae6d7dd-0afa-452a-8510-fcbc86097eda","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:36.806Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"1ae6d7dd-0afa-452a-8510-fcbc86097eda","keyPrefix":"sk-oai-18pN6HgMefs6...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:36 | 201 |    1.618401ms |      100.64.0.4 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:36.902Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a2dea043-7539-4b9f-91fc-2fa380c4294e","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:36.902Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"a2dea043-7539-4b9f-91fc-2fa380c4294e","keyPrefix":"sk-oai-1SNTBMNwFt3P...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:36 | 201 |    1.656953ms |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:36.931Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"14da871d-2114-413c-b78a-5c1e75609f8f","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:36.931Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"14da871d-2114-413c-b78a-5c1e75609f8f","keyPrefix":"sk-oai-1YoJzYa030X0...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:36 | 201 |    1.685434ms |      100.64.0.4 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:36.959Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"5c887e69-c3ef-4d59-b453-0b5281f26dc2","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:36.959Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"5c887e69-c3ef-4d59-b453-0b5281f26dc2","keyPrefix":"sk-oai-qa5tJSXbjH0t...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:36 | 201 |    1.787851ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:36.986Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"a2dea043-7539-4b9f-91fc-2fa380c4294e"}
{"level":"info","timestamp":"2026-05-21T01:07:36.986Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"a2dea043-7539-4b9f-91fc-2fa380c4294e","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:36 | 200 |    1.875686ms |      100.64.0.3 | DELETE   "/v1/api-keys/a2dea043-7539-4b9f-91fc-2fa380c4294e"
{"level":"info","timestamp":"2026-05-21T01:07:37.012Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"14da871d-2114-413c-b78a-5c1e75609f8f"}
{"level":"info","timestamp":"2026-05-21T01:07:37.012Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"14da871d-2114-413c-b78a-5c1e75609f8f","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:37 | 200 |    1.936277ms |      100.64.0.4 | DELETE   "/v1/api-keys/14da871d-2114-413c-b78a-5c1e75609f8f"
{"level":"info","timestamp":"2026-05-21T01:07:37.038Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"5c887e69-c3ef-4d59-b453-0b5281f26dc2"}
{"level":"info","timestamp":"2026-05-21T01:07:37.038Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"5c887e69-c3ef-4d59-b453-0b5281f26dc2","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:07:37 | 200 |    1.866538ms |      100.64.0.3 | DELETE   "/v1/api-keys/5c887e69-c3ef-4d59-b453-0b5281f26dc2"
{"level":"info","timestamp":"2026-05-21T01:07:37.066Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"880dfa3b-89fe-49f8-b17a-0145c9d2f75e","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:37.066Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"880dfa3b-89fe-49f8-b17a-0145c9d2f75e","keyPrefix":"sk-oai-rlIgwuNOnJxD...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:37 | 201 |    1.568699ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:37.092Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f2bd0aa0-39cc-46d4-bf08-981ea1cb80f7","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:37.092Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f2bd0aa0-39cc-46d4-bf08-981ea1cb80f7","keyPrefix":"sk-oai-aDnAOpzrbz7y...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:37 | 201 |    1.538445ms |      100.64.0.4 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:37.121Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"83d8bd69-7a93-4fcf-bad1-4d090f058077","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:37.121Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"83d8bd69-7a93-4fcf-bad1-4d090f058077","keyPrefix":"sk-oai-11Smyz7yKJ50...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:37 | 201 |    1.496817ms |      100.64.0.3 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:37.413Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"48c26437-4641-455c-95e5-eff4eb6c6dc3","ephemeral":true}
{"level":"info","timestamp":"2026-05-21T01:07:37.414Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"48c26437-4641-455c-95e5-eff4eb6c6dc3","keyPrefix":"sk-oai-2m5wcHP2Jcxf...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true}
[GIN] 2026/05/21 - 01:07:37 | 201 |     1.71885ms |      100.64.0.4 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:37 | 200 |    3.680079ms |      100.64.0.3 | POST     "/v1/api-keys/search"
[GIN] 2026/05/21 - 01:07:37 | 200 |     595.576µs |      100.64.0.2 | POST     "/v1/api-keys/search"
{"level":"info","timestamp":"2026-05-21T01:07:37.497Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"afb65003-045e-479c-a2fe-b5028f4a00e6","ephemeral":true}
{"level":"info","timestamp":"2026-05-21T01:07:37.497Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"afb65003-045e-479c-a2fe-b5028f4a00e6","keyPrefix":"sk-oai-KWtkVeVQ7jFJ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true}
[GIN] 2026/05/21 - 01:07:37 | 201 |     1.72361ms |      100.64.0.2 | POST     "/v1/api-keys"
{"level":"info","timestamp":"2026-05-21T01:07:37.849Z","caller":"api_keys/service.go:305","message":"Ephemeral key cleanup completed","deletedCount":0}
[GIN] 2026/05/21 - 01:07:37 | 200 |     781.385µs |             ::1 | POST     "/internal/v1/api-keys/cleanup"
[GIN] 2026/05/21 - 01:07:37 | 200 |     396.153µs |      100.64.0.4 | GET      "/v1/api-keys/afb65003-045e-479c-a2fe-b5028f4a00e6"
[GIN] 2026/05/21 - 01:07:38 | 200 |       44.59µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:07:38.567Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"07a2fe45-fd91-46f2-b4e0-67c777e6eafb","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:38.567Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"07a2fe45-fd91-46f2-b4e0-67c777e6eafb","keyPrefix":"sk-oai-KruE5iE2RCMH...","username":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:38 | 201 |    1.721489ms |      100.64.0.3 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:43 | 200 |      44.674µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:45 | 200 |      44.039µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:48 | 200 |       38.42µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:53 | 200 |      39.652µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:07:55 | 200 |      43.929µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:07:57.691Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"42bc4bb3-2267-4856-9d01-a9ed1bd247bc","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:07:57.691Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"42bc4bb3-2267-4856-9d01-a9ed1bd247bc","keyPrefix":"sk-oai-1XJph1jGdEnm...","username":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:07:57 | 201 |    1.996958ms |      100.64.0.2 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:07:58 | 200 |      40.571µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:03 | 200 |      34.071µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:05 | 200 |      40.345µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:08 | 200 |      33.027µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:13 | 200 |      38.197µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:15 | 200 |      39.374µs |      10.134.0.2 | GET      "/health"
{"level":"warn","timestamp":"2026-05-21T01:08:16.909Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-failed-sa","requestedSubscription":"e2e-apikey-failed-sub","error":"requested model is unhealthy in subscription"}
{"level":"error","timestamp":"2026-05-21T01:08:16.909Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/05/21 - 01:08:16 | 403 |     308.924µs |      100.64.0.4 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:08:18 | 200 |      32.874µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:23 | 200 |      32.876µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:25 | 200 |      36.489µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:28 | 200 |      33.679µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:33 | 200 |      32.233µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:35 | 200 |      31.673µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:08:36.190Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"7fdd6985-b609-4e44-a28e-80b26f8c18d3","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:08:36.190Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"7fdd6985-b609-4e44-a28e-80b26f8c18d3","keyPrefix":"sk-oai-o8t0pqhsgRMM...","username":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:08:36 | 201 |    3.009506ms |      100.64.0.3 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:08:38 | 200 |      31.003µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:43 | 200 |      35.298µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:45 | 200 |      32.724µs |      10.134.0.2 | GET      "/health"
{"level":"warn","timestamp":"2026-05-21T01:08:46.182Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-unreconciled-sa","requestedSubscription":"e2e-apikey-unreconciled-sub","error":"requested model is unhealthy in subscription"}
{"level":"error","timestamp":"2026-05-21T01:08:46.182Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/05/21 - 01:08:46 | 400 |     214.515µs |      100.64.0.2 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:08:48 | 200 |       32.57µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:53 | 200 |      37.474µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:55 | 200 |       33.16µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:08:58 | 200 |      33.913µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:03 | 200 |      34.472µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:05 | 200 |      38.555µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:09:06.717Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"9dda64f8-34fa-4fbb-a4f9-c247e435224f","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:09:06.717Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"9dda64f8-34fa-4fbb-a4f9-c247e435224f","keyPrefix":"sk-oai-UGqybgVVI0gj...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:09:06 | 201 |    1.834776ms |      100.64.0.4 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:09:06 | 200 |     940.736µs |     10.132.0.32 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/21 - 01:09:06 | 200 |     204.077µs |      100.64.0.4 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/21 - 01:09:08 | 200 |      32.219µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:13 | 200 |      34.748µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:15 | 200 |      36.771µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:18 | 200 |        34.9µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:23 | 200 |      46.354µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:23 | 200 |     821.079µs |     10.132.0.32 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/05/21 - 01:09:23 | 200 |       94.56µs |      100.64.0.3 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/05/21 - 01:09:25 | 200 |      33.431µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:28 | 200 |      35.343µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:33 | 200 |      43.467µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:35 | 200 |      41.232µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:38 | 200 |      39.659µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:43 | 200 |      41.488µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:45 | 200 |      42.499µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:48 | 200 |      44.602µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:53 | 200 |      41.704µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:55 | 200 |      38.469µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:09:58 | 200 |      41.223µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:03 | 200 |       40.78µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:05 | 200 |      39.903µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:08 | 200 |      37.292µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:13 | 200 |      41.885µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:15 | 200 |      41.001µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:18 | 200 |      40.444µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:23 | 200 |      43.928µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:25 | 200 |      62.943µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:28 | 200 |      38.689µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:33 | 200 |      40.603µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:35 | 200 |      45.499µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:38 | 200 |      41.405µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:43 | 200 |      64.751µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:45 | 200 |      39.357µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:48 | 200 |      40.907µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:53 | 200 |      41.271µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:55 | 200 |      37.192µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:10:58 | 200 |      39.932µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:03 | 200 |       41.83µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:05 | 200 |      38.131µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:08 | 200 |      41.861µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:13 | 200 |      39.551µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:15 | 200 |      53.207µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:18 | 200 |      39.251µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:23 | 200 |      36.082µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:25 | 200 |      35.046µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:28 | 200 |      37.357µs |      10.134.0.2 | GET      "/health"
[GIN] 2026/05/21 - 01:11:33 | 200 |      31.903µs |      10.134.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-05-21T01:11:33.176Z","caller":"api_keys/store_postgres.go:460","message":"Revoked API key","id":"9dda64f8-34fa-4fbb-a4f9-c247e435224f"}
{"level":"info","timestamp":"2026-05-21T01:11:33.176Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"9dda64f8-34fa-4fbb-a4f9-c247e435224f","revokedBy":"system:serviceaccount:default:tester-regular-user"}
[GIN] 2026/05/21 - 01:11:33 | 200 |    3.352075ms |      100.64.0.2 | DELETE   "/v1/api-keys/9dda64f8-34fa-4fbb-a4f9-c247e435224f"
{"level":"info","timestamp":"2026-05-21T01:11:33.212Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"347b8de1-68c0-432e-a5f6-ab0ce78e217c","ephemeral":false}
{"level":"info","timestamp":"2026-05-21T01:11:33.212Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"347b8de1-68c0-432e-a5f6-ab0ce78e217c","keyPrefix":"sk-oai-1bdGvQDdlE2x...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/05/21 - 01:11:33 | 201 |     1.49684ms |      100.64.0.3 | POST     "/v1/api-keys"
[GIN] 2026/05/21 - 01:11:35 | 200 |      46.095µs |      10.134.0.2 | GET      "/health"