{"level":"info","timestamp":"2026-05-21T11:55:42.880Z","caller":"cmd/main.go:65","message":"Loading database connection URL from secret..."} {"level":"info","timestamp":"2026-05-21T11:55:42.896Z","caller":"cmd/main.go:174","message":"Connecting to PostgreSQL database..."} {"level":"info","timestamp":"2026-05-21T11:55:42.896Z","caller":"cmd/main.go:103","message":"Metrics server starting","address":":9090"} {"level":"info","timestamp":"2026-05-21T11:55:43.032Z","caller":"api_keys/db_driver.go:90","message":"Database schema applied","version":3} {"level":"info","timestamp":"2026-05-21T11:55:43.032Z","caller":"api_keys/db_driver.go:57","message":"Connected to PostgreSQL database (schema applied)"} {"level":"info","timestamp":"2026-05-21T11:55:43.137Z","caller":"cmd/main.go:195","message":"Resolved gateway internal host for access probes","host":"maas-default-gateway-openshift-default.openshift-ingress.svc.cluster.local"} {"level":"info","timestamp":"2026-05-21T11:55:43.171Z","caller":"cmd/main.go:136","message":"Server starting","address":":8443","secure":true} [GIN] 2026/05/21 - 11:55:48 | 200 | 69.511µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:53 | 200 | 36.714µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:55:58 | 200 | 45.648µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:03 | 200 | 40.747µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:08 | 200 | 43.518µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:13 | 200 | 34.236µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:18 | 200 | 35.751µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:20 | 200 | 44.507µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:23 | 200 | 36.326µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:28 | 200 | 33.422µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:30 | 200 | 35.324µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:33 | 200 | 37.951µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:38 | 200 | 34.143µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:40 | 200 | 40µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:43 | 200 | 36.598µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:48 | 200 | 34.52µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:50 | 200 | 34.39µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:53 | 200 | 35.183µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:56:58 | 200 | 33.054µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:00 | 200 | 35.095µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:03 | 200 | 39.423µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:08 | 200 | 33.778µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:10 | 200 | 34.914µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:13 | 200 | 33.373µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:18 | 200 | 36.218µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:20 | 200 | 34.57µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:23 | 200 | 43.851µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:24 | 200 | 20.232µs | 100.64.0.3 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:57:26.651Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"b47b0ccd-f143-4411-a1a9-b7a569cf9fd7","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:26.652Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b47b0ccd-f143-4411-a1a9-b7a569cf9fd7","keyPrefix":"sk-oai-15W8xjipQTuf...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:26 | 201 | 10.316959ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:26 | 200 | 955.715µs | 100.64.0.2 | GET "/v1/api-keys/b47b0ccd-f143-4411-a1a9-b7a569cf9fd7" {"level":"info","timestamp":"2026-05-21T11:57:26.731Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"8e5ece8b-6d50-4f77-99e7-84b15d2421d8","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:26.731Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"8e5ece8b-6d50-4f77-99e7-84b15d2421d8","keyPrefix":"sk-oai-14P8J7BbFgOK...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:26 | 201 | 930.072µs | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:26.762Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"5ff712de-8ec7-4c5a-b835-7cc64cc47746","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:26.762Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"5ff712de-8ec7-4c5a-b835-7cc64cc47746","keyPrefix":"sk-oai-16hG8h2bNCZc...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:26 | 201 | 946.182µs | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:26 | 200 | 6.184304ms | 100.64.0.2 | POST "/v1/api-keys/search" [GIN] 2026/05/21 - 11:57:26 | 200 | 351.79µs | 100.64.0.4 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:57:26.870Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3a14dedf-4a81-4871-b758-2c2d7956a338","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:26.870Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3a14dedf-4a81-4871-b758-2c2d7956a338","keyPrefix":"sk-oai-15iwgkQNtDRT...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:26 | 201 | 912.154µs | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:26.905Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"3a14dedf-4a81-4871-b758-2c2d7956a338"} {"level":"info","timestamp":"2026-05-21T11:57:26.905Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"3a14dedf-4a81-4871-b758-2c2d7956a338","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:26 | 200 | 1.394199ms | 100.64.0.4 | DELETE "/v1/api-keys/3a14dedf-4a81-4871-b758-2c2d7956a338" [GIN] 2026/05/21 - 11:57:26 | 200 | 298.045µs | 100.64.0.3 | GET "/v1/api-keys/3a14dedf-4a81-4871-b758-2c2d7956a338" {"level":"info","timestamp":"2026-05-21T11:57:26.987Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"9a5116ae-d51f-4ce4-8871-9c3497ab36fd","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:26.987Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"9a5116ae-d51f-4ce4-8871-9c3497ab36fd","keyPrefix":"sk-oai-uzIvcY5WuNRK...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:26 | 201 | 945.004µs | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:27 | 200 | 329.345µs | 100.64.0.3 | GET "/v1/api-keys/9a5116ae-d51f-4ce4-8871-9c3497ab36fd" [GIN] 2026/05/21 - 11:57:27 | 200 | 3.932018ms | 100.64.0.4 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:57:27.073Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"9a5116ae-d51f-4ce4-8871-9c3497ab36fd"} {"level":"info","timestamp":"2026-05-21T11:57:27.073Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"9a5116ae-d51f-4ce4-8871-9c3497ab36fd","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"} [GIN] 2026/05/21 - 11:57:27 | 200 | 1.161877ms | 100.64.0.2 | DELETE "/v1/api-keys/9a5116ae-d51f-4ce4-8871-9c3497ab36fd" {"level":"info","timestamp":"2026-05-21T11:57:27.104Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"id":"7c110f51-4cb7-4fcf-8bb1-e0644f1248e2","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.104Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"7c110f51-4cb7-4fcf-8bb1-e0644f1248e2","keyPrefix":"sk-oai-1evoHie0iXfa...","username":"system:serviceaccount:maas-admin:tester-admin-user","groups":["system:serviceaccounts","system:serviceaccounts:maas-admin","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 970.369µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"warn","timestamp":"2026-05-21T11:57:27.133Z","caller":"api_keys/handler.go:124","message":"Unauthorized API key access attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"7c110f51-4cb7-4fcf-8bb1-e0644f1248e2"} [GIN] 2026/05/21 - 11:57:27 | 404 | 294.052µs | 100.64.0.3 | GET "/v1/api-keys/7c110f51-4cb7-4fcf-8bb1-e0644f1248e2" {"level":"warn","timestamp":"2026-05-21T11:57:27.159Z","caller":"api_keys/handler.go:305","message":"Unauthorized API key revocation attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","keyOwner":"system:serviceaccount:maas-admin:tester-admin-user","keyId":"7c110f51-4cb7-4fcf-8bb1-e0644f1248e2"} [GIN] 2026/05/21 - 11:57:27 | 404 | 317.445µs | 100.64.0.4 | DELETE "/v1/api-keys/7c110f51-4cb7-4fcf-8bb1-e0644f1248e2" {"level":"info","timestamp":"2026-05-21T11:57:27.191Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"5c208a6d-2b04-4532-adc3-566c81bfd03b","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.191Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"5c208a6d-2b04-4532-adc3-566c81bfd03b","keyPrefix":"sk-oai-zaTbZBLK2iaD...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 964.675µs | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.219Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"cf8e4802-ffd0-45e3-a0a2-90707ce00b4b","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.219Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"cf8e4802-ffd0-45e3-a0a2-90707ce00b4b","keyPrefix":"sk-oai-PFsixwkJVlsU...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 924.985µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.256Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"68a8040f-8664-4228-9b14-5ef392d508cf","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.256Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"68a8040f-8664-4228-9b14-5ef392d508cf","keyPrefix":"sk-oai-OskS7SpETlYx...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 1.356152ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:27 | 200 | 259.29µs | 100.64.0.4 | GET "/v1/api-keys/5c208a6d-2b04-4532-adc3-566c81bfd03b" {"level":"info","timestamp":"2026-05-21T11:57:27.312Z","caller":"api_keys/store_postgres.go:447","message":"Revoked all keys for user","count":6,"user":"system:serviceaccount:default:tester-regular-user"} {"level":"info","timestamp":"2026-05-21T11:57:27.312Z","caller":"api_keys/handler.go:531","message":"Bulk revoked API keys","count":6,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:27 | 200 | 1.330847ms | 100.64.0.3 | POST "/v1/api-keys/bulk-revoke" [GIN] 2026/05/21 - 11:57:27 | 200 | 239.772µs | 100.64.0.2 | GET "/v1/api-keys/5c208a6d-2b04-4532-adc3-566c81bfd03b" [GIN] 2026/05/21 - 11:57:27 | 200 | 242.241µs | 100.64.0.3 | GET "/v1/api-keys/cf8e4802-ffd0-45e3-a0a2-90707ce00b4b" [GIN] 2026/05/21 - 11:57:27 | 200 | 269.506µs | 100.64.0.4 | GET "/v1/api-keys/68a8040f-8664-4228-9b14-5ef392d508cf" {"level":"warn","timestamp":"2026-05-21T11:57:27.428Z","caller":"api_keys/handler.go:508","message":"Unauthorized bulk revoke attempt","requestingUser":"system:serviceaccount:default:tester-regular-user","targetUser":"someotheruser"} [GIN] 2026/05/21 - 11:57:27 | 403 | 104.305µs | 100.64.0.2 | POST "/v1/api-keys/bulk-revoke" {"level":"info","timestamp":"2026-05-21T11:57:27.459Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fd0a6346-1dc3-4361-8a72-6e125da71807","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.459Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"fd0a6346-1dc3-4361-8a72-6e125da71807","keyPrefix":"sk-oai-FLXHo15nCndf...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 986.381µs | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:27 | 200 | 255.558µs | 100.64.0.4 | GET "/v1/api-keys/fd0a6346-1dc3-4361-8a72-6e125da71807" {"level":"info","timestamp":"2026-05-21T11:57:27.527Z","caller":"api_keys/store_postgres.go:447","message":"Revoked all keys for user","count":1,"user":"system:serviceaccount:default:tester-regular-user"} {"level":"info","timestamp":"2026-05-21T11:57:27.527Z","caller":"api_keys/handler.go:531","message":"Bulk revoked API keys","count":1,"targetUser":"system:serviceaccount:default:tester-regular-user","revokedBy":"system:serviceaccount:maas-admin:tester-admin-user"} [GIN] 2026/05/21 - 11:57:27 | 200 | 923.276µs | 100.64.0.2 | POST "/v1/api-keys/bulk-revoke" {"level":"info","timestamp":"2026-05-21T11:57:27.560Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"4691ae23-9857-489e-9cb8-465629ec9449","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.560Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"4691ae23-9857-489e-9cb8-465629ec9449","keyPrefix":"sk-oai-yRxyxGk9xXyy...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 962.511µs | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.590Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fbe86381-7f41-4c75-bf25-2e78eb4be11b","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.590Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"fbe86381-7f41-4c75-bf25-2e78eb4be11b","keyPrefix":"sk-oai-ExdtXwl4zanJ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 1.084236ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"error","timestamp":"2026-05-21T11:57:27.619Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested expiration (4320h0m0s) exceeds maximum allowed (90 days): expiration exceeds maximum allowed","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/05/21 - 11:57:27 | 400 | 203.425µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.648Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"56b3fcfd-20be-445f-95f3-4186189c7b59","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.648Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"56b3fcfd-20be-445f-95f3-4186189c7b59","keyPrefix":"sk-oai-19bTwqCOx6wW...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 960.168µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.682Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"49af6f26-0e53-4c5a-863b-ac52c54bb5e3","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.682Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"49af6f26-0e53-4c5a-863b-ac52c54bb5e3","keyPrefix":"sk-oai-1NaxBsWMiZZD...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 1.317148ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.712Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"770dfaad-88e1-4e5f-9520-03ceda692703","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.712Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"770dfaad-88e1-4e5f-9520-03ceda692703","keyPrefix":"sk-oai-1BAs3bOHMRVy...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 1.113676ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:27 | 200 | 11.727632ms | 100.64.0.4 | GET "/v1/models" {"level":"info","timestamp":"2026-05-21T11:57:27.906Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"84f247f7-1ff0-42e3-ab66-7713eea3325b","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:27.906Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"84f247f7-1ff0-42e3-ab66-7713eea3325b","keyPrefix":"sk-oai-bEDoPkqcVo27...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:27 | 201 | 1.26121ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:27.955Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"84f247f7-1ff0-42e3-ab66-7713eea3325b"} {"level":"info","timestamp":"2026-05-21T11:57:27.955Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"84f247f7-1ff0-42e3-ab66-7713eea3325b","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:27 | 200 | 1.17872ms | 100.64.0.3 | DELETE "/v1/api-keys/84f247f7-1ff0-42e3-ab66-7713eea3325b" [GIN] 2026/05/21 - 11:57:28 | 200 | 41.37µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:57:30.047Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"37a57a7a-387f-4ef9-a50c-a1baf17d04b3","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.048Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"37a57a7a-387f-4ef9-a50c-a1baf17d04b3","keyPrefix":"sk-oai-Qlx3CeIEbEQz...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 1.158043ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.076Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"37a57a7a-387f-4ef9-a50c-a1baf17d04b3"} {"level":"info","timestamp":"2026-05-21T11:57:30.076Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"37a57a7a-387f-4ef9-a50c-a1baf17d04b3","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:30 | 200 | 1.071794ms | 100.64.0.2 | DELETE "/v1/api-keys/37a57a7a-387f-4ef9-a50c-a1baf17d04b3" [GIN] 2026/05/21 - 11:57:30 | 404 | 361.994µs | 100.64.0.3 | DELETE "/v1/api-keys/37a57a7a-387f-4ef9-a50c-a1baf17d04b3" [GIN] 2026/05/21 - 11:57:30 | 404 | 213.81µs | 100.64.0.3 | DELETE "/v1/api-keys/nonexistent-uuid-12345" {"level":"info","timestamp":"2026-05-21T11:57:30.171Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f78cb1fa-d80d-4ad5-bd83-868941bdb812","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.171Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f78cb1fa-d80d-4ad5-bd83-868941bdb812","keyPrefix":"sk-oai-4BUD5uYRHeAQ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 828.696µs | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.197Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"f78cb1fa-d80d-4ad5-bd83-868941bdb812"} {"level":"info","timestamp":"2026-05-21T11:57:30.197Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"f78cb1fa-d80d-4ad5-bd83-868941bdb812","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:30 | 200 | 1.085981ms | 100.64.0.2 | DELETE "/v1/api-keys/f78cb1fa-d80d-4ad5-bd83-868941bdb812" {"level":"info","timestamp":"2026-05-21T11:57:30.223Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"64a3a669-c507-445a-89f8-47a9d90426b8","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.223Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"64a3a669-c507-445a-89f8-47a9d90426b8","keyPrefix":"sk-oai-1DiFQkmGqkJN...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 758.812µs | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.325Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f8b31b9c-aee4-4136-ab52-2e58abef8c92","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.325Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f8b31b9c-aee4-4136-ab52-2e58abef8c92","keyPrefix":"sk-oai-10UK92dVHSsR...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 925.406µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.353Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"11166510-090a-4823-90a6-e734ad539415","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.353Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"11166510-090a-4823-90a6-e734ad539415","keyPrefix":"sk-oai-13hg5Lg9aVXT...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 1.008578ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.383Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"584e194f-1047-4d23-a288-ffaea44188c0","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.383Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"584e194f-1047-4d23-a288-ffaea44188c0","keyPrefix":"sk-oai-1CvBLGdZ101y...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 842.358µs | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.412Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"f8b31b9c-aee4-4136-ab52-2e58abef8c92"} {"level":"info","timestamp":"2026-05-21T11:57:30.412Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"f8b31b9c-aee4-4136-ab52-2e58abef8c92","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:30 | 200 | 957.535µs | 100.64.0.4 | DELETE "/v1/api-keys/f8b31b9c-aee4-4136-ab52-2e58abef8c92" {"level":"info","timestamp":"2026-05-21T11:57:30.439Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"11166510-090a-4823-90a6-e734ad539415"} {"level":"info","timestamp":"2026-05-21T11:57:30.439Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"11166510-090a-4823-90a6-e734ad539415","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:30 | 200 | 975.459µs | 100.64.0.2 | DELETE "/v1/api-keys/11166510-090a-4823-90a6-e734ad539415" {"level":"info","timestamp":"2026-05-21T11:57:30.467Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"584e194f-1047-4d23-a288-ffaea44188c0"} {"level":"info","timestamp":"2026-05-21T11:57:30.467Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"584e194f-1047-4d23-a288-ffaea44188c0","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:57:30 | 200 | 1.127698ms | 100.64.0.3 | DELETE "/v1/api-keys/584e194f-1047-4d23-a288-ffaea44188c0" {"level":"info","timestamp":"2026-05-21T11:57:30.497Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f0b77f53-637d-4458-9c4f-6dbfc7533d49","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.497Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f0b77f53-637d-4458-9c4f-6dbfc7533d49","keyPrefix":"sk-oai-dYQBCRIV1A7i...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 891.453µs | 100.64.0.3 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.525Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"cad4a442-4e94-4541-b462-f12152a891db","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.525Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"cad4a442-4e94-4541-b462-f12152a891db","keyPrefix":"sk-oai-sERiRKGMPHKk...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 874.059µs | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:30 | 200 | 30.419µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:57:30.551Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"b300557d-82d4-4a57-893e-083196f3426f","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:30.551Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b300557d-82d4-4a57-893e-083196f3426f","keyPrefix":"sk-oai-EW1cMSU5vipc...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:30 | 201 | 736.564µs | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:30.861Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"75f83863-6fd2-4d87-8885-cc5a7c79d1f1","ephemeral":true} {"level":"info","timestamp":"2026-05-21T11:57:30.861Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"75f83863-6fd2-4d87-8885-cc5a7c79d1f1","keyPrefix":"sk-oai-38gKFW5YTN7W...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true} [GIN] 2026/05/21 - 11:57:30 | 201 | 1.188167ms | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:30 | 200 | 5.757815ms | 100.64.0.4 | POST "/v1/api-keys/search" [GIN] 2026/05/21 - 11:57:30 | 200 | 490.001µs | 100.64.0.4 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:57:30.956Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"56442b70-9f32-4da5-9abe-737491902618","ephemeral":true} {"level":"info","timestamp":"2026-05-21T11:57:30.956Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"56442b70-9f32-4da5-9abe-737491902618","keyPrefix":"sk-oai-NfzgLxAoRzkZ...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":true} [GIN] 2026/05/21 - 11:57:30 | 201 | 921.164µs | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:57:31.377Z","caller":"api_keys/service.go:305","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/05/21 - 11:57:31 | 200 | 686.344µs | ::1 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/05/21 - 11:57:31 | 200 | 309.623µs | 100.64.0.3 | GET "/v1/api-keys/56442b70-9f32-4da5-9abe-737491902618" {"level":"info","timestamp":"2026-05-21T11:57:32.106Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"324e6af2-aaba-421b-a440-f409b7b9541a","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:32.106Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"324e6af2-aaba-421b-a440-f409b7b9541a","keyPrefix":"sk-oai-H0MeEqUTSJfW...","username":"system:serviceaccount:llm:e2e-apikey-active-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:32 | 201 | 1.23954ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:33 | 200 | 38.347µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:38 | 200 | 45.82µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:40 | 200 | 39.348µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:43 | 200 | 41.262µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:48 | 200 | 41.918µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:50 | 200 | 56.682µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:57:51.259Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"bca90912-72fc-4f12-a7ff-a2ca4da24966","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:57:51.259Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"bca90912-72fc-4f12-a7ff-a2ca4da24966","keyPrefix":"sk-oai-1MPXWQkxhNO9...","username":"system:serviceaccount:llm:e2e-apikey-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:57:51 | 201 | 1.316403ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:57:53 | 200 | 37.062µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:57:58 | 200 | 34.794µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:00 | 200 | 35.578µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:03 | 200 | 36.065µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:08 | 200 | 34.559µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:10 | 200 | 34.191µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-05-21T11:58:10.561Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-failed-sa","requestedSubscription":"e2e-apikey-failed-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-05-21T11:58:10.561Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/05/21 - 11:58:10 | 403 | 259.958µs | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:58:13 | 200 | 35.959µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:18 | 200 | 34.091µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:20 | 200 | 32.871µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:23 | 200 | 34.662µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:28 | 200 | 44.011µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:58:29.895Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"da15a83b-efdd-4c19-a6a2-3b0378ff0910","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:58:29.895Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"da15a83b-efdd-4c19-a6a2-3b0378ff0910","keyPrefix":"sk-oai-1eNO1SGQNj91...","username":"system:serviceaccount:llm:e2e-apikey-pending-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:58:29 | 201 | 1.611759ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:58:30 | 200 | 31.963µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:33 | 200 | 35.954µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:38 | 200 | 34.333µs | 10.134.0.2 | GET "/health" {"level":"warn","timestamp":"2026-05-21T11:58:40.222Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:llm:e2e-apikey-unreconciled-sa","requestedSubscription":"e2e-apikey-unreconciled-sub","error":"requested model is unhealthy in subscription"} {"level":"error","timestamp":"2026-05-21T11:58:40.222Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"requested model is unhealthy in subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/05/21 - 11:58:40 | 400 | 256.242µs | 100.64.0.3 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:58:40 | 200 | 34.67µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:43 | 200 | 39.656µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:48 | 200 | 39.707µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:50 | 200 | 43.502µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:53 | 200 | 38.388µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:58:58 | 200 | 35.046µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:00 | 200 | 34.9µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:03 | 200 | 34.398µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:59:04.125Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-d446091d","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"bb8c3939-fcbe-4f2b-bdc9-b62b349531fa","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:59:04.125Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"bb8c3939-fcbe-4f2b-bdc9-b62b349531fa","keyPrefix":"sk-oai-ACbEf84dNDG4...","username":"system:serviceaccount:llm:e2e-filter-sa-d446091d","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:59:04 | 201 | 1.75454ms | 100.64.0.2 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:59:04.156Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-d446091d","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"81b39485-64d8-4fc7-870b-23f0f5dcf6c9","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:59:04.156Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"81b39485-64d8-4fc7-870b-23f0f5dcf6c9","keyPrefix":"sk-oai-eNeeKhQ0yNhK...","username":"system:serviceaccount:llm:e2e-filter-sa-d446091d","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:59:04 | 201 | 1.623669ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:59:04.191Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-filter-sa-d446091d","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"09ab89c2-bfed-41fc-9981-94b3189f1a6e","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:59:04.191Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"09ab89c2-bfed-41fc-9981-94b3189f1a6e","keyPrefix":"sk-oai-1Hk3pCYU14rH...","username":"system:serviceaccount:llm:e2e-filter-sa-d446091d","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:59:04 | 201 | 1.625395ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:59:04 | 200 | 4.442254ms | 100.64.0.3 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:59:04.256Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"bb8c3939-fcbe-4f2b-bdc9-b62b349531fa"} {"level":"info","timestamp":"2026-05-21T11:59:04.256Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"bb8c3939-fcbe-4f2b-bdc9-b62b349531fa","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-d446091d"} [GIN] 2026/05/21 - 11:59:04 | 200 | 1.736872ms | 100.64.0.4 | DELETE "/v1/api-keys/bb8c3939-fcbe-4f2b-bdc9-b62b349531fa" {"level":"info","timestamp":"2026-05-21T11:59:04.288Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"81b39485-64d8-4fc7-870b-23f0f5dcf6c9"} {"level":"info","timestamp":"2026-05-21T11:59:04.289Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"81b39485-64d8-4fc7-870b-23f0f5dcf6c9","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-d446091d"} [GIN] 2026/05/21 - 11:59:04 | 200 | 1.750629ms | 100.64.0.2 | DELETE "/v1/api-keys/81b39485-64d8-4fc7-870b-23f0f5dcf6c9" {"level":"info","timestamp":"2026-05-21T11:59:04.316Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"09ab89c2-bfed-41fc-9981-94b3189f1a6e"} {"level":"info","timestamp":"2026-05-21T11:59:04.316Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"09ab89c2-bfed-41fc-9981-94b3189f1a6e","revokedBy":"system:serviceaccount:llm:e2e-filter-sa-d446091d"} [GIN] 2026/05/21 - 11:59:04 | 200 | 1.572416ms | 100.64.0.3 | DELETE "/v1/api-keys/09ab89c2-bfed-41fc-9981-94b3189f1a6e" [GIN] 2026/05/21 - 11:59:08 | 200 | 38.077µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:10 | 200 | 34.541µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T11:59:13.354Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"b976af2b-bf39-429d-9fe0-72b803d4def7","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:59:13.354Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b976af2b-bf39-429d-9fe0-72b803d4def7","keyPrefix":"sk-oai-19NCnZRJfFM5...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:59:13 | 201 | 1.867927ms | 100.64.0.4 | POST "/v1/api-keys" {"level":"info","timestamp":"2026-05-21T11:59:13.383Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d13d0735-d159-483e-af83-dff9f39dd0da","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:59:13.383Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d13d0735-d159-483e-af83-dff9f39dd0da","keyPrefix":"sk-oai-8t0NPofkQaiu...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:59:13 | 201 | 1.348845ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:59:13 | 200 | 3.684954ms | 100.64.0.3 | POST "/v1/api-keys/search" {"level":"info","timestamp":"2026-05-21T11:59:13.446Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"b976af2b-bf39-429d-9fe0-72b803d4def7"} {"level":"info","timestamp":"2026-05-21T11:59:13.446Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"b976af2b-bf39-429d-9fe0-72b803d4def7","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:59:13 | 200 | 1.584332ms | 100.64.0.3 | DELETE "/v1/api-keys/b976af2b-bf39-429d-9fe0-72b803d4def7" {"level":"info","timestamp":"2026-05-21T11:59:13.473Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"d13d0735-d159-483e-af83-dff9f39dd0da"} {"level":"info","timestamp":"2026-05-21T11:59:13.473Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"d13d0735-d159-483e-af83-dff9f39dd0da","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 11:59:13 | 200 | 1.905741ms | 100.64.0.4 | DELETE "/v1/api-keys/d13d0735-d159-483e-af83-dff9f39dd0da" {"level":"info","timestamp":"2026-05-21T11:59:13.506Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f56497e2-0e59-412f-8798-99d1886e7b92","ephemeral":false} {"level":"info","timestamp":"2026-05-21T11:59:13.506Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f56497e2-0e59-412f-8798-99d1886e7b92","keyPrefix":"sk-oai-1HBroMyWGtNb...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 11:59:13 | 201 | 1.415199ms | 100.64.0.2 | POST "/v1/api-keys" [GIN] 2026/05/21 - 11:59:13 | 200 | 624.517µs | 10.132.0.27 | POST "/internal/v1/api-keys/validate" [GIN] 2026/05/21 - 11:59:13 | 200 | 188.377µs | 100.64.0.2 | POST "/internal/v1/subscriptions/select" [GIN] 2026/05/21 - 11:59:13 | 200 | 30.984µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:18 | 200 | 43.449µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:20 | 200 | 42.424µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:23 | 200 | 40.4µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:28 | 200 | 44.722µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:30 | 200 | 572.441µs | 10.132.0.27 | POST "/internal/v1/api-keys/validate" [GIN] 2026/05/21 - 11:59:30 | 200 | 99.267µs | 100.64.0.3 | POST "/internal/v1/subscriptions/select" [GIN] 2026/05/21 - 11:59:30 | 200 | 47.058µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:33 | 200 | 38.157µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:38 | 200 | 44.695µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:40 | 200 | 40.972µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:43 | 200 | 39.454µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:48 | 200 | 47.723µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:50 | 200 | 39.627µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:53 | 200 | 45.83µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 11:59:58 | 200 | 49.843µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:00 | 200 | 39.501µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:03 | 200 | 33.65µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T12:00:05.426Z","caller":"api_keys/service.go:305","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/05/21 - 12:00:05 | 200 | 528.944µs | 10.133.0.37 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/05/21 - 12:00:08 | 200 | 41.196µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:10 | 200 | 60.426µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:13 | 200 | 38.552µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:18 | 200 | 33.123µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:20 | 200 | 34.843µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:23 | 200 | 40.452µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:28 | 200 | 41.522µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:30 | 200 | 35.028µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:33 | 200 | 40.139µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:38 | 200 | 34.866µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:40 | 200 | 39.513µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:43 | 200 | 52.094µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:48 | 200 | 39.404µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:50 | 200 | 45.426µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:53 | 200 | 40.704µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:00:58 | 200 | 34.724µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:00 | 200 | 37.283µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:03 | 200 | 36.228µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:08 | 200 | 45.808µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:10 | 200 | 34.194µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:13 | 200 | 45.519µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:18 | 200 | 33.429µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:20 | 200 | 35.614µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:23 | 200 | 36.129µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:28 | 200 | 33.145µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:30 | 200 | 40.62µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:33 | 200 | 33.681µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:38 | 200 | 42.593µs | 10.134.0.2 | GET "/health" [GIN] 2026/05/21 - 12:01:40 | 200 | 34.798µs | 10.134.0.2 | GET "/health" {"level":"info","timestamp":"2026-05-21T12:01:41.942Z","caller":"api_keys/store_postgres.go:467","message":"Revoked API key","id":"f56497e2-0e59-412f-8798-99d1886e7b92"} {"level":"info","timestamp":"2026-05-21T12:01:41.942Z","caller":"api_keys/handler.go:326","message":"Revoked API key","keyId":"f56497e2-0e59-412f-8798-99d1886e7b92","revokedBy":"system:serviceaccount:default:tester-regular-user"} [GIN] 2026/05/21 - 12:01:41 | 200 | 1.835758ms | 100.64.0.4 | DELETE "/v1/api-keys/f56497e2-0e59-412f-8798-99d1886e7b92" {"level":"info","timestamp":"2026-05-21T12:01:41.987Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"639cc7db-cc8c-495a-809f-698283b98081","ephemeral":false} {"level":"info","timestamp":"2026-05-21T12:01:41.987Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"639cc7db-cc8c-495a-809f-698283b98081","keyPrefix":"sk-oai-1CeXt0I5ZMDq...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/05/21 - 12:01:41 | 201 | 1.58896ms | 100.64.0.4 | POST "/v1/api-keys" [GIN] 2026/05/21 - 12:01:43 | 200 | 34.845µs | 10.134.0.2 | GET "/health"