{"level":"info","timestamp":"2026-05-21T11:49:27.126Z","caller":"cmd/main.go:65","message":"Loading database connection URL from secret..."} {"level":"info","timestamp":"2026-05-21T11:49:27.144Z","caller":"cmd/main.go:174","message":"Connecting to PostgreSQL database..."} {"level":"info","timestamp":"2026-05-21T11:49:27.212Z","caller":"cmd/main.go:103","message":"Metrics server starting","address":":9090"} {"level":"info","timestamp":"2026-05-21T11:49:27.229Z","caller":"api_keys/db_driver.go:90","message":"Database schema applied","version":3} {"level":"info","timestamp":"2026-05-21T11:49:27.229Z","caller":"api_keys/db_driver.go:57","message":"Connected to PostgreSQL database (schema applied)"} {"level":"info","timestamp":"2026-05-21T11:49:27.335Z","caller":"cmd/main.go:195","message":"Resolved gateway internal host for access probes","host":"maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local"} {"level":"info","timestamp":"2026-05-21T11:49:27.416Z","caller":"cmd/main.go:136","message":"Server starting","address":":8443","secure":true} [GIN] 2026/05/21 - 11:49:33 | 200 | 57.496µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:49:38 | 200 | 42.375µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:49:43 | 200 | 33.455µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:49:48 | 200 | 56.817µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:49:53 | 200 | 32.635µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:49:58 | 200 | 31.995µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:03 | 200 | 59.755µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:05 | 200 | 39.931µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:08 | 200 | 41.196µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:13 | 200 | 33.489µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:15 | 200 | 42.727µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:18 | 200 | 43.414µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:23 | 200 | 33.183µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:25 | 200 | 32.969µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:28 | 200 | 39.684µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:33 | 200 | 45.064µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:35 | 200 | 33.186µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:38 | 200 | 36.006µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:43 | 200 | 33.375µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:45 | 200 | 34.767µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:48 | 200 | 36.627µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:53 | 200 | 57.376µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:55 | 200 | 39.008µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:50:58 | 200 | 34.835µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:03 | 200 | 41.332µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:05 | 200 | 34.351µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:08 | 200 | 37.177µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:13 | 200 | 39.735µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:14 | 200 | 19.002µs | 100.64.0.4 | GET "/health" [GIN] 2026/05/21 - 11:51:15 | 200 | 49.986µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:51:15.439Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"23153c30-8124-49c7-9527-af3e7dc332da","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:15.439Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"23153c30-8124-49c7-9527-af3e7dc332da","keyPrefix":"sk-oai-1G6bOCQdylxr...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:15 | 201 | 11.597013ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:15 | 200 | 1.48899ms | 100.64.0.2 | GET "/v1/api-keys/23153c30-8124-49c7-9527-af3e7dc332da" {"level":"info","timestamp":"2026-05-21T11:51:15.519Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"dac69e7a-f3e5-4fa6-b1c8-bceb111a7ae5","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:15.519Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"dac69e7a-f3e5-4fa6-b1c8-bceb111a7ae5","keyPrefix":"sk-oai-1Ook4a0ykmt6...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:15 | 201 | 1.856709ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:15.557Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"8f3c1b0e-c2cb-4b93-a9e0-67acecd3c72b","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:15.557Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"8f3c1b0e-c2cb-4b93-a9e0-67acecd3c72b","keyPrefix":"sk-oai-fFWBynw6g33A...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:15 | 201 | 1.767221ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:15 | 200 | 7.23803ms | 100.64.0.2 | POST "/v1/api-keys/search" [GIN] 2026/05/21 - 11:51:15 | 200 | 654.2µs | 100.64.0.3 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:51:15.674Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"ad1f401e-4ea9-457f-a783-5d401f28578e","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:15.675Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"ad1f401e-4ea9-457f-a783-5d401f28578e","keyPrefix":"sk-oai-f6VbSnYpKk1t...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:15 | 201 | 1.73573ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:15.705Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"ad1f401e-4ea9-457f-a783-5d401f28578e"} {"level":"info","timestamp":"2026-05-21T11:51:15.705Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"ad1f401e-4ea9-457f-a783-5d401f28578e","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:15 | 200 | 2.677547ms | 100.64.0.3 | DELETE "/v1/api-keys/ad1f401e-4ea9-457f-a783-5d401f28578e" [GIN] 2026/05/21 - 11:51:15 | 200 | 550.564µs | 100.64.0.2 | GET "/v1/api-keys/ad1f401e-4ea9-457f-a783-5d401f28578e" {"level":"info","timestamp":"2026-05-21T11:51:15.767Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"df45171a-4065-41d7-9214-a71f96b538b2","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:15.767Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"df45171a-4065-41d7-9214-a71f96b538b2","keyPrefix":"sk-oai-hGg1wRKm9hIF...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:15 | 201 | 1.741018ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:15 | 200 | 576.05µs | 100.64.0.4 | GET "/v1/api-keys/df45171a-4065-41d7-9214-a71f96b538b2" [GIN] 2026/05/21 - 11:51:15 | 200 | 4.602626ms | 100.64.0.4 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:51:15.855Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"df45171a-4065-41d7-9214-a71f96b538b2"} {"level":"info","timestamp":"2026-05-21T11:51:15.855Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"df45171a-4065-41d7-9214-a71f96b538b2","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"} [GIN] 2026/05/21 - 11:51:15 | 200 | 2.484467ms | 100.64.0.2 | DELETE "/v1/api-keys/df45171a-4065-41d7-9214-a71f96b538b2" {"level":"info","timestamp":"2026-05-21T11:51:15.887Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"id":"26e0749c-5160-4dd8-b2e6-9df2d696d0b2","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:15.887Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"26e0749c-5160-4dd8-b2e6-9df2d696d0b2","keyPrefix":"sk-oai-EBSAWJfNd7Zc...","username":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:15 | 201 | 1.894973ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"warn","timestamp":"2026-05-21T11:51:15.915Z","caller":"api_keys/handler.go:124","message":"Unauthorized API key access attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"26e0749c-5160-4dd8-b2e6-9df2d696d0b2"} [GIN] 2026/05/21 - 11:51:15 | 404 | 504.327µs | 100.64.0.4 | GET "/v1/api-keys/26e0749c-5160-4dd8-b2e6-9df2d696d0b2" {"level":"warn","timestamp":"2026-05-21T11:51:15.963Z","caller":"api_keys/handler.go:305","message":"Unauthorized API key revocation attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"26e0749c-5160-4dd8-b2e6-9df2d696d0b2"} [GIN] 2026/05/21 - 11:51:15 | 404 | 687.034µs | 100.64.0.2 | DELETE "/v1/api-keys/26e0749c-5160-4dd8-b2e6-9df2d696d0b2" {"level":"info","timestamp":"2026-05-21T11:51:16.003Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"ecf8f700-265b-41a9-8f44-6f61d6baf080","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.003Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"ecf8f700-265b-41a9-8f44-6f61d6baf080","keyPrefix":"sk-oai-PAhzIEhPv1eP...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.787153ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.042Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e8c1be15-6568-4097-a9ea-41ced3e87ede","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.042Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e8c1be15-6568-4097-a9ea-41ced3e87ede","keyPrefix":"sk-oai-1Tt0QBOx3ORJ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.794071ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.070Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"50ba9762-4dd8-4fc3-88b4-01e92e38ef56","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.070Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"50ba9762-4dd8-4fc3-88b4-01e92e38ef56","keyPrefix":"sk-oai-XP9WuDduq2KY...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.766838ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:16 | 200 | 468.302µs | 100.64.0.3 | GET "/v1/api-keys/ecf8f700-265b-41a9-8f44-6f61d6baf080" {"level":"info","timestamp":"2026-05-21T11:51:16.125Z","caller":"api_keys/store_postgres.go:447","message":"Revoked all keys for user","count":6,"user":"system:serviceaccount:default:tester-regular-user"} {"level":"info","timestamp":"2026-05-21T11:51:16.125Z","caller":"api_keys/handler.go:531","message":"Bulk revoked API keys","count":6,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:16 | 200 | 2.144445ms | 100.64.0.4 | POST "/v1/api-keys/bulk-revoke" [GIN] 2026/05/21 - 11:51:16 | 200 | 472.701µs | 100.64.0.2 | GET "/v1/api-keys/ecf8f700-265b-41a9-8f44-6f61d6baf080" [GIN] 2026/05/21 - 11:51:16 | 200 | 437.124µs | 100.64.0.2 | GET "/v1/api-keys/e8c1be15-6568-4097-a9ea-41ced3e87ede" [GIN] 2026/05/21 - 11:51:16 | 200 | 524.693µs | 100.64.0.3 | GET "/v1/api-keys/50ba9762-4dd8-4fc3-88b4-01e92e38ef56" {"level":"warn","timestamp":"2026-05-21T11:51:16.243Z","caller":"api_keys/handler.go:508","message":"Unauthorized bulk revoke attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","targetUser":"someotheruser"} [GIN] 2026/05/21 - 11:51:16 | 403 | 125.012µs | 100.64.0.4 | POST "/v1/api-keys/bulk-revoke" {"level":"info","timestamp":"2026-05-21T11:51:16.275Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"261e2ec4-b1ff-493d-945d-44a6975932c2","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.275Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"261e2ec4-b1ff-493d-945d-44a6975932c2","keyPrefix":"sk-oai-GroZB4nBPcfi...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.777586ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:16 | 200 | 441.983µs | 100.64.0.3 | GET "/v1/api-keys/261e2ec4-b1ff-493d-945d-44a6975932c2" {"level":"info","timestamp":"2026-05-21T11:51:16.335Z","caller":"api_keys/store_postgres.go:447","message":"Revoked all keys for user","count":1,"user":"system:serviceaccount:default:tester-regular-user"} {"level":"info","timestamp":"2026-05-21T11:51:16.335Z","caller":"api_keys/handler.go:531","message":"Bulk revoked API keys","count":1,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"} [GIN] 2026/05/21 - 11:51:16 | 200 | 1.672948ms | 100.64.0.3 | POST "/v1/api-keys/bulk-revoke" {"level":"info","timestamp":"2026-05-21T11:51:16.365Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fc978f68-b4e9-47ad-a06b-6d8c1d973108","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.365Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"fc978f68-b4e9-47ad-a06b-6d8c1d973108","keyPrefix":"sk-oai-1TrrPaQ69s2L...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.638384ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.409Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"27f096a8-e4ef-44de-8f04-faae98e8b0f0","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.409Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"27f096a8-e4ef-44de-8f04-faae98e8b0f0","keyPrefix":"sk-oai-1NMTggqqzjyt...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.606314ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"error","timestamp":"2026-05-21T11:51:16.436Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested expiration (4320h0m0s) exceeds maximum allowed (90 days): expiration exceeds maximum allowed","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/05/21 - 11:51:16 | 400 | 169.209µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.465Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"a111d912-14c4-4396-af22-44721c2f2100","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.465Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"a111d912-14c4-4396-af22-44721c2f2100","keyPrefix":"sk-oai-E7dD9sbqC7ie...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.772558ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.495Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"23aebcbe-331d-4ea7-bcb7-4e420d519bc3","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.495Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"23aebcbe-331d-4ea7-bcb7-4e420d519bc3","keyPrefix":"sk-oai-bsr8Bzcv2ulA...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.944311ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.525Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"8bcd4bc8-4525-4f8f-ba23-4ce7ac0a3791","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.525Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"8bcd4bc8-4525-4f8f-ba23-4ce7ac0a3791","keyPrefix":"sk-oai-UqIwl2KerZer...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.64801ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:16 | 200 | 11.864548ms | 100.64.0.2 | GET "/v1/models" {"level":"info","timestamp":"2026-05-21T11:51:16.714Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"6b34a30e-537f-405b-8456-863c8b3acf6b","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:16.714Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"6b34a30e-537f-405b-8456-863c8b3acf6b","keyPrefix":"sk-oai-hInKcmpGlyVx...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:16 | 201 | 1.819956ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:16.761Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"6b34a30e-537f-405b-8456-863c8b3acf6b"} {"level":"info","timestamp":"2026-05-21T11:51:16.761Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"6b34a30e-537f-405b-8456-863c8b3acf6b","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:16 | 200 | 2.351229ms | 100.64.0.4 | DELETE "/v1/api-keys/6b34a30e-537f-405b-8456-863c8b3acf6b" [GIN] 2026/05/21 - 11:51:18 | 200 | 48.71µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:51:18.854Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d0eb265e-f7a6-4b67-bf22-ef7eeeaf2e64","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:18.854Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d0eb265e-f7a6-4b67-bf22-ef7eeeaf2e64","keyPrefix":"sk-oai-70ocFG79oVGU...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:18 | 201 | 2.319522ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:18.884Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"d0eb265e-f7a6-4b67-bf22-ef7eeeaf2e64"} {"level":"info","timestamp":"2026-05-21T11:51:18.884Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"d0eb265e-f7a6-4b67-bf22-ef7eeeaf2e64","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:18 | 200 | 1.983657ms | 100.64.0.4 | DELETE "/v1/api-keys/d0eb265e-f7a6-4b67-bf22-ef7eeeaf2e64" [GIN] 2026/05/21 - 11:51:18 | 404 | 803.361µs | 100.64.0.2 | DELETE "/v1/api-keys/d0eb265e-f7a6-4b67-bf22-ef7eeeaf2e64" [GIN] 2026/05/21 - 11:51:18 | 404 | 398.863µs | 100.64.0.3 | DELETE "/v1/api-keys/nonexistent-uuid-12345" {"level":"info","timestamp":"2026-05-21T11:51:18.974Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"b480bd3f-3508-49e5-9fab-bb3a44f7966a","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:18.974Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b480bd3f-3508-49e5-9fab-bb3a44f7966a","keyPrefix":"sk-oai-1anzFNjDnPKJ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:18 | 201 | 1.642184ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.004Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"b480bd3f-3508-49e5-9fab-bb3a44f7966a"} {"level":"info","timestamp":"2026-05-21T11:51:19.004Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"b480bd3f-3508-49e5-9fab-bb3a44f7966a","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:19 | 200 | 2.151382ms | 100.64.0.4 | DELETE "/v1/api-keys/b480bd3f-3508-49e5-9fab-bb3a44f7966a" {"level":"info","timestamp":"2026-05-21T11:51:19.034Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2136c741-4444-4a12-bda4-c4dd62c380fb","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.034Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"2136c741-4444-4a12-bda4-c4dd62c380fb","keyPrefix":"sk-oai-yoGD62PnybPn...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.597728ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.151Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"05dc520b-49a4-41d3-b996-25f36033a6ac","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.151Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"05dc520b-49a4-41d3-b996-25f36033a6ac","keyPrefix":"sk-oai-up5r9l3lHo9Z...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.768706ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.178Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"5a580d09-98ad-4f55-b070-8226b4bbfcbc","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.178Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"5a580d09-98ad-4f55-b070-8226b4bbfcbc","keyPrefix":"sk-oai-1A9vxpDKaF7I...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.582144ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.205Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"eae14622-b371-47ae-8aa5-acc97dc2ec1e","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.205Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"eae14622-b371-47ae-8aa5-acc97dc2ec1e","keyPrefix":"sk-oai-kCU8jdC4NJGh...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.741248ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.233Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"05dc520b-49a4-41d3-b996-25f36033a6ac"} {"level":"info","timestamp":"2026-05-21T11:51:19.233Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"05dc520b-49a4-41d3-b996-25f36033a6ac","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:19 | 200 | 2.07765ms | 100.64.0.2 | DELETE "/v1/api-keys/05dc520b-49a4-41d3-b996-25f36033a6ac" {"level":"info","timestamp":"2026-05-21T11:51:19.264Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"5a580d09-98ad-4f55-b070-8226b4bbfcbc"} {"level":"info","timestamp":"2026-05-21T11:51:19.264Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"5a580d09-98ad-4f55-b070-8226b4bbfcbc","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:19 | 200 | 2.018884ms | 100.64.0.3 | DELETE "/v1/api-keys/5a580d09-98ad-4f55-b070-8226b4bbfcbc" {"level":"info","timestamp":"2026-05-21T11:51:19.294Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"eae14622-b371-47ae-8aa5-acc97dc2ec1e"} {"level":"info","timestamp":"2026-05-21T11:51:19.294Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"eae14622-b371-47ae-8aa5-acc97dc2ec1e","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:51:19 | 200 | 1.907598ms | 100.64.0.3 | DELETE "/v1/api-keys/eae14622-b371-47ae-8aa5-acc97dc2ec1e" {"level":"info","timestamp":"2026-05-21T11:51:19.324Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"9c99af2a-1853-4578-a61c-0923e9439914","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.324Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"9c99af2a-1853-4578-a61c-0923e9439914","keyPrefix":"sk-oai-1FD6vgUfTXfL...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.660124ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.351Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"738295c1-7c06-4a65-be0b-7af09ef6d290","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.351Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"738295c1-7c06-4a65-be0b-7af09ef6d290","keyPrefix":"sk-oai-llyuw0kQmMQq...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.627206ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.379Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"33e347b7-ce53-4667-892a-ea1f20e2c3ef","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:19.379Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"33e347b7-ce53-4667-892a-ea1f20e2c3ef","keyPrefix":"sk-oai-3NrATiwN25VA...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.695196ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:19.678Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"9a406348-dfad-4829-9d9c-feb5dfe972c8","ephemeral":true} {"level":"info","timestamp":"2026-05-21T11:51:19.678Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"9a406348-dfad-4829-9d9c-feb5dfe972c8","keyPrefix":"sk-oai-oP98utZJfS5F...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.772969ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:19 | 200 | 4.818532ms | 100.64.0.3 | POST "/v1/api-keys/search" [GIN] 2026/05/21 - 11:51:19 | 200 | 840.272µs | 100.64.0.4 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:51:19.775Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"bd9c0011-a47e-40f5-8f3e-218f1bb8eac1","ephemeral":true} {"level":"info","timestamp":"2026-05-21T11:51:19.775Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"bd9c0011-a47e-40f5-8f3e-218f1bb8eac1","keyPrefix":"sk-oai-17PiYlO1sLJU...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true} [GIN] 2026/05/21 - 11:51:19 | 201 | 1.736951ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:51:20.219Z","caller":"api_keys/service.go:305","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/05/21 - 11:51:20 | 200 | 855.643µs | ::1 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/05/21 - 11:51:20 | 200 | 453.04µs | 100.64.0.2 | GET "/v1/api-keys/bd9c0011-a47e-40f5-8f3e-218f1bb8eac1" {"level":"info","timestamp":"2026-05-21T11:51:20.939Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"1220ca41-9a91-4fdb-98e2-d70b0a1b38c0","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:20.939Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"1220ca41-9a91-4fdb-98e2-d70b0a1b38c0","keyPrefix":"sk-oai-1L4nLaEIhcHI...","username":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:20 | 201 | 1.771711ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:23 | 200 | 66.796µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:25 | 200 | 39.926µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:28 | 200 | 41.264µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:33 | 200 | 40.54µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:35 | 200 | 44.86µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:38 | 200 | 41.912µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:51:40.201Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"3d9364f6-4511-4f22-b409-759fd2087191","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:51:40.201Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3d9364f6-4511-4f22-b409-759fd2087191","keyPrefix":"sk-oai-fjHGNziVHpBl...","username":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:51:40 | 201 | 2.143285ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:51:43 | 200 | 41.735µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:45 | 200 | 41.212µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:48 | 200 | 66.955µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:53 | 200 | 41.196µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:55 | 200 | 47.748µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:51:58 | 200 | 33.775µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-05-21T11:51:59.562Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-failed-sa","requestedSubscription":"e2e-apikey-failed-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-05-21T11:51:59.562Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/05/21 - 11:51:59 | 403 | 250.886µs | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:52:03 | 200 | 40.518µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:05 | 200 | 47.938µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:08 | 200 | 39.248µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:13 | 200 | 39.976µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:15 | 200 | 33.725µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:18 | 200 | 40.673µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:52:18.920Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"60dbda10-5a3b-420d-8e52-c888f7d73dfc","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:52:18.920Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"60dbda10-5a3b-420d-8e52-c888f7d73dfc","keyPrefix":"sk-oai-RxwesBKl61RU...","username":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:52:18 | 201 | 1.969428ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:52:23 | 200 | 36.339µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:25 | 200 | 39.861µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:28 | 200 | 41.032µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-05-21T11:52:28.888Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-unreconciled-sa","requestedSubscription":"e2e-apikey-unreconciled-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-05-21T11:52:28.888Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/05/21 - 11:52:28 | 400 | 266.109µs | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:52:33 | 200 | 55.61µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:35 | 200 | 36.51µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:38 | 200 | 34.109µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:43 | 200 | 33.191µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:45 | 200 | 32.667µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:48 | 200 | 36.721µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:52:53 | 200 | 33.87µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:52:55.281Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-3596b670","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"0cf0356c-5e63-4769-b56a-81af04a67062","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:52:55.281Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"0cf0356c-5e63-4769-b56a-81af04a67062","keyPrefix":"sk-oai-YNCgd7WyPxEk...","username":"system:serviceaccount:llm:e2e-filter-sa-3596b670","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:52:55 | 201 | 1.66242ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:52:55.314Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-3596b670","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"976f3cd6-637b-4300-98d0-29da2ff79947","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:52:55.314Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"976f3cd6-637b-4300-98d0-29da2ff79947","keyPrefix":"sk-oai-c2ErHqThjH6v...","username":"system:serviceaccount:llm:e2e-filter-sa-3596b670","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:52:55 | 201 | 1.359258ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:52:55.343Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-3596b670","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"6ff6a9e0-e05b-478c-a85b-abac43184b70","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:52:55.343Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"6ff6a9e0-e05b-478c-a85b-abac43184b70","keyPrefix":"sk-oai-1XhLSS7VES9J...","username":"system:serviceaccount:llm:e2e-filter-sa-3596b670","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:52:55 | 201 | 1.107095ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:52:55 | 200 | 5.165681ms | 100.64.0.2 | POST "/v1/api-keys/search" [GIN] 2026/05/21 - 11:52:55 | 200 | 31.108µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:52:55.431Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"0cf0356c-5e63-4769-b56a-81af04a67062"} {"level":"info","timestamp":"2026-05-21T11:52:55.431Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"0cf0356c-5e63-4769-b56a-81af04a67062","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-3596b670"} [GIN] 2026/05/21 - 11:52:55 | 200 | 1.596417ms | 100.64.0.3 | DELETE "/v1/api-keys/0cf0356c-5e63-4769-b56a-81af04a67062" {"level":"info","timestamp":"2026-05-21T11:52:55.462Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"976f3cd6-637b-4300-98d0-29da2ff79947"} {"level":"info","timestamp":"2026-05-21T11:52:55.462Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"976f3cd6-637b-4300-98d0-29da2ff79947","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-3596b670"} [GIN] 2026/05/21 - 11:52:55 | 200 | 1.632075ms | 100.64.0.4 | DELETE "/v1/api-keys/976f3cd6-637b-4300-98d0-29da2ff79947" {"level":"info","timestamp":"2026-05-21T11:52:55.492Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"6ff6a9e0-e05b-478c-a85b-abac43184b70"} {"level":"info","timestamp":"2026-05-21T11:52:55.492Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"6ff6a9e0-e05b-478c-a85b-abac43184b70","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-3596b670"} [GIN] 2026/05/21 - 11:52:55 | 200 | 1.530517ms | 100.64.0.2 | DELETE "/v1/api-keys/6ff6a9e0-e05b-478c-a85b-abac43184b70" [GIN] 2026/05/21 - 11:52:58 | 200 | 32.945µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:03 | 200 | 35.321µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:53:04.609Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"67b97597-eecc-4d5a-87b7-b546f925e1c8","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:53:04.609Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"67b97597-eecc-4d5a-87b7-b546f925e1c8","keyPrefix":"sk-oai-CpEOJcIqs5h0...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:53:04 | 201 | 1.536135ms | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:53:04.638Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3611c7e7-e3f2-4fd5-b7ce-1ce313947ef9","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:53:04.638Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3611c7e7-e3f2-4fd5-b7ce-1ce313947ef9","keyPrefix":"sk-oai-tFqq46d6mu4F...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:53:04 | 201 | 1.267317ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:53:04 | 200 | 4.646298ms | 100.64.0.2 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:53:04.699Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"67b97597-eecc-4d5a-87b7-b546f925e1c8"} {"level":"info","timestamp":"2026-05-21T11:53:04.699Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"67b97597-eecc-4d5a-87b7-b546f925e1c8","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:53:04 | 200 | 1.70255ms | 100.64.0.3 | DELETE "/v1/api-keys/67b97597-eecc-4d5a-87b7-b546f925e1c8" {"level":"info","timestamp":"2026-05-21T11:53:04.726Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"3611c7e7-e3f2-4fd5-b7ce-1ce313947ef9"} {"level":"info","timestamp":"2026-05-21T11:53:04.726Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"3611c7e7-e3f2-4fd5-b7ce-1ce313947ef9","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:53:04 | 200 | 1.678022ms | 100.64.0.4 | DELETE "/v1/api-keys/3611c7e7-e3f2-4fd5-b7ce-1ce313947ef9" {"level":"info","timestamp":"2026-05-21T11:53:04.760Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"36cb6c95-b6c3-4e63-baf3-42296499d9eb","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:53:04.760Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"36cb6c95-b6c3-4e63-baf3-42296499d9eb","keyPrefix":"sk-oai-1MDySHAR56Ys...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:53:04 | 201 | 1.326841ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:53:05 | 200 | 1.133222ms | 10.132.0.33 | POST "/internal/v1/api-keys/validate" [GIN] 2026/05/21 - 11:53:05 | 200 | 219.051µs | 100.64.0.4 | POST "/internal/v1/subscriptions/select" [GIN] 2026/05/21 - 11:53:05 | 200 | 32.705µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:08 | 200 | 35.626µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:13 | 200 | 36.307µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:15 | 200 | 35.485µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:18 | 200 | 36.928µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:21 | 200 | 1.547284ms | 10.132.0.33 | POST "/internal/v1/api-keys/validate" [GIN] 2026/05/21 - 11:53:21 | 200 | 103.983µs | 100.64.0.2 | POST "/internal/v1/subscriptions/select" [GIN] 2026/05/21 - 11:53:23 | 200 | 33.511µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:25 | 200 | 41.356µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:28 | 200 | 50.558µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:33 | 200 | 34.527µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:35 | 200 | 33.059µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:38 | 200 | 35.743µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:43 | 200 | 35.966µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:45 | 200 | 40.941µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:48 | 200 | 41.009µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:53 | 200 | 35.688µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:55 | 200 | 36.887µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:53:58 | 200 | 31.972µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:03 | 200 | 52.204µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:05 | 200 | 34.377µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:08 | 200 | 33.715µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:13 | 200 | 33.228µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:15 | 200 | 34.241µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:18 | 200 | 36.158µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:23 | 200 | 36.101µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:25 | 200 | 32.579µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:28 | 200 | 35.443µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:33 | 200 | 33.82µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:35 | 200 | 34.427µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:38 | 200 | 49.852µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:43 | 200 | 31.983µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:45 | 200 | 33.274µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:48 | 200 | 35.287µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:53 | 200 | 33.983µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:55 | 200 | 35.989µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:54:58 | 200 | 33.371µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:03 | 200 | 33.313µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:05 | 200 | 36.266µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:08 | 200 | 33.827µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:13 | 200 | 35.689µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:15 | 200 | 32.881µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:18 | 200 | 35.903µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:23 | 200 | 35.527µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:25 | 200 | 37.757µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:28 | 200 | 34.74µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:33 | 200 | 36.685µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:55:33.922Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"36cb6c95-b6c3-4e63-baf3-42296499d9eb"} {"level":"info","timestamp":"2026-05-21T11:55:33.922Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"36cb6c95-b6c3-4e63-baf3-42296499d9eb","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:55:33 | 200 | 2.622385ms | 100.64.0.3 | DELETE "/v1/api-keys/36cb6c95-b6c3-4e63-baf3-42296499d9eb" {"level":"info","timestamp":"2026-05-21T11:55:33.959Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1af363fa-604b-4806-9ae1-7ed053ec38f7","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:55:33.959Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"1af363fa-604b-4806-9ae1-7ed053ec38f7","keyPrefix":"sk-oai-1N4eAtGV5ibE...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:55:33 | 201 | 1.233827ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:55:35 | 200 | 34.319µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:38 | 200 | 33.655µs | 10.134.0.2 | GET "/health"