[GIN] 2026/06/12 - 19:42:32 | 200 |     551.527µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:42:32 | 200 |     265.417µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:42:32 | 200 |     505.123µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:42:32 | 200 |     275.982µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:42:35 | 200 |      34.076µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:42:40 | 200 |       33.34µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:42:40 | 200 |      72.379µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:42:40 | 200 |     767.244µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:42:40 | 200 |     176.295µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:42:45 | 200 |      33.772µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:42:50 | 200 |      34.577µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:42:50 | 200 |      37.118µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:42:55 | 200 |       38.08µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:42:57.532Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"4d344565-7bf6-4ece-bbc9-e7f62e0d43b0","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:42:57.532Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"4d344565-7bf6-4ece-bbc9-e7f62e0d43b0","keyPrefix":"sk-oai-8snAF9RMj66F...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:42:57 | 201 |    2.020935ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:42:57 | 200 |     562.779µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:42:57 | 200 |     152.641µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:43:00 | 200 |      40.567µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:00 | 200 |      37.612µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:05 | 200 |      48.057µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:10 | 200 |      43.502µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:10 | 200 |      48.469µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:15 | 200 |      40.966µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:20 | 200 |      43.746µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:20 | 200 |      40.774µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:22 | 200 |     780.557µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:43:22 | 200 |     148.156µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:43:25 | 200 |      39.338µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:30 | 200 |      43.556µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:30 | 200 |       46.28µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:35 | 200 |      43.893µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:38 | 200 |     831.392µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:43:38 | 200 |     156.929µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:43:40 | 200 |      37.939µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:40 | 200 |      38.856µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:45 | 200 |      36.336µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:50 | 200 |      38.106µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:50 | 200 |      47.064µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:55 | 200 |      40.401µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:43:55 | 200 |     991.447µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:43:55 | 200 |     357.482µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:44:00 | 200 |      45.068µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:00 | 200 |      38.428µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:05 | 200 |      42.086µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:10 | 200 |      42.802µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:10 | 200 |      68.641µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:12 | 200 |   10.642526ms |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:44:12 | 200 |      126.29µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:44:15 | 200 |      40.237µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:20 | 200 |      39.139µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:20 | 200 |      37.096µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:21 | 200 |     985.899µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:44:21 | 200 |     202.713µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:44:25 | 200 |      40.205µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:30 | 200 |      44.605µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:30 | 200 |      39.707µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:35 | 200 |      53.798µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:44:38.353Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-ordering-sa-53ea92","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"32de1770-3760-4d78-9384-b6a106113367","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:44:38.353Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"32de1770-3760-4d78-9384-b6a106113367","keyPrefix":"sk-oai-1e7txV8wV9c8...","username":"system:serviceaccount:models-as-a-service:e2e-ordering-sa-53ea92","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:44:38 | 201 |    2.456369ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:44:38 | 200 |     711.651µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:44:38 | 200 |     146.842µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:44:38 | 200 |     701.078µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:44:38 | 200 |     248.408µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:44:40 | 200 |      38.688µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:40 | 200 |     920.184µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:44:40 | 200 |     152.385µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:44:40 | 200 |       42.13µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:45 | 200 |      38.316µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:50 | 200 |      47.355µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:50 | 200 |      37.711µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:44:55 | 200 |      31.759µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:00 | 200 |      40.284µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:00 | 200 |      40.887µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:45:04.690Z","caller":"api_keys/service.go:305","message":"Ephemeral key cleanup completed","deletedCount":0}
[GIN] 2026/06/12 - 19:45:04 | 200 |    1.241444ms |     10.134.0.38 | POST     "/internal/v1/api-keys/cleanup"
[GIN] 2026/06/12 - 19:45:05 | 200 |      43.793µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:10 | 200 |      59.154µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:10 | 200 |      44.769µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:15 | 200 |      38.423µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:45:19.402Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-success","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"6ca2b341-8317-4623-9f10-4e53d3c2c274","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:45:19.402Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"6ca2b341-8317-4623-9f10-4e53d3c2c274","keyPrefix":"sk-oai-TXCJ09KPYvqS...","username":"system:serviceaccount:models-as-a-service:e2e-sa-success","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:45:19 | 201 |    2.223961ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:45:19 | 200 |     878.012µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:45:19 | 200 |     365.263µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:45:20 | 200 |      42.671µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:20 | 200 |       40.68µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:25 | 200 |      38.563µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:45:28.493Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-no-sub","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"b035d547-3b39-4761-8378-b145ca41aad6","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:45:28.493Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b035d547-3b39-4761-8378-b145ca41aad6","keyPrefix":"sk-oai-qYieRrcoOA1V...","username":"system:serviceaccount:models-as-a-service:e2e-sa-no-sub","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:45:28 | 201 |    2.105282ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:45:30 | 200 |      60.108µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:30 | 200 |      43.539µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:35 | 200 |      45.926µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:36 | 200 |     788.845µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:45:36 | 200 |     212.975µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:45:40 | 200 |      32.974µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:40 | 200 |      43.506µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:45 | 200 |      39.208µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:45:46.148Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-sa-with-sub","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"2098f262-aa3a-4eea-965e-386c3c5f6a8f","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:45:46.148Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"2098f262-aa3a-4eea-965e-386c3c5f6a8f","keyPrefix":"sk-oai-1b3CB6qGzieJ...","username":"system:serviceaccount:llm:e2e-sa-with-sub","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:45:46 | 201 |    2.342975ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:45:46 | 200 |     675.986µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:45:46 | 200 |     150.326µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:45:50 | 200 |      38.338µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:50 | 200 |      42.612µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:45:55 | 200 |      38.266µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:00 | 200 |      39.293µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:00 | 200 |      43.843µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:46:03.697Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-single-sub","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"7defcd27-27c0-4709-a915-e8c61137fc27","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:46:03.697Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"7defcd27-27c0-4709-a915-e8c61137fc27","keyPrefix":"sk-oai-1PjnCbR17j5n...","username":"system:serviceaccount:models-as-a-service:e2e-sa-single-sub","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:46:03 | 201 |    2.083573ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:46:03 | 200 |     552.445µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:46:03 | 200 |     134.276µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:46:05 | 200 |       41.15µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:10 | 200 |      52.323µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:10 | 200 |      40.833µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:15 | 200 |      52.447µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:20 | 200 |      40.347µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:20 | 200 |      39.298µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:46:21.053Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"d6eefd65-e382-475d-a5cc-b9d233f01de8","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:46:21.053Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d6eefd65-e382-475d-a5cc-b9d233f01de8","keyPrefix":"sk-oai-1PEJSTxaTiWd...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:46:21 | 201 |    2.245411ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-06-12T19:46:21.090Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"5fc83448-d569-4967-b451-2274d63e73f0","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:46:21.090Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"5fc83448-d569-4967-b451-2274d63e73f0","keyPrefix":"sk-oai-Kj6SRYMeNA7b...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:46:21 | 201 |    1.708627ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:46:21 | 200 |      530.39µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:46:21 | 200 |     146.183µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:46:21 | 200 |     536.433µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:46:21 | 200 |     163.217µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:46:25 | 200 |      42.966µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:30 | 200 |      48.872µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:30 | 200 |      37.064µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:35 | 200 |       38.42µs |      10.133.0.2 | GET      "/health"
{"level":"warn","timestamp":"2026-06-12T19:46:38.811Z","caller":"api_keys/service.go:130","message":"Subscription selection failed when creating API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-user","requestedSubscription":"e2e-test-other-subscription","error":"access denied to requested subscription"}
{"level":"error","timestamp":"2026-06-12T19:46:38.811Z","caller":"api_keys/handler.go:192","message":"Failed to create API key","error":"access denied to requested subscription","stacktrace":"github.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys.(*Handler).CreateAPIKey\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/api_keys/handler.go:192\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:121\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/06/12 - 19:46:38 | 400 |     265.212µs |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:46:40 | 200 |       63.63µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:40 | 200 |      53.916µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:45 | 200 |      39.261µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:50 | 200 |      50.412µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:50 | 200 |      38.784µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:46:55 | 200 |      39.087µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:46:56.205Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"2c409443-188a-4665-a5c6-b2933874b5ca","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:46:56.205Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"2c409443-188a-4665-a5c6-b2933874b5ca","keyPrefix":"sk-oai-MvLHdFJF0p41...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:46:56 | 201 |    2.011154ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:46:56 | 200 |     524.468µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:46:56 | 200 |     160.092µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:47:00 | 200 |      54.112µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:00 | 200 |      38.949µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:05 | 200 |      39.323µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:47:05.294Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-auth-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"3e70251c-4d12-4e13-a182-3a3c92def476","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:47:05.294Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3e70251c-4d12-4e13-a182-3a3c92def476","keyPrefix":"sk-oai-YKskvxvnXfyr...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-auth-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:47:05 | 201 |    2.169976ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:47:10 | 200 |      55.439µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:10 | 200 |      40.315µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:13 | 200 |     795.114µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:47:13 | 200 |     111.301µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:47:15 | 200 |      41.263µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:20 | 200 |      42.879µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:20 | 200 |      38.421µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:47:22.754Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"22b61f96-3d4a-4454-a8cb-10f46d9526be","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:47:22.754Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"22b61f96-3d4a-4454-a8cb-10f46d9526be","keyPrefix":"sk-oai-LRxmkGbgQCYn...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:47:22 | 201 |    2.163461ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:47:22 | 200 |     535.036µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:47:22 | 200 |     156.059µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:47:25 | 200 |      43.963µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:30 | 200 |      46.815µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:30 | 200 |      59.981µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:35 | 200 |      39.343µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:40 | 200 |      41.683µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:40 | 200 |      54.269µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:45 | 200 |      41.984µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:50 | 200 |      40.595µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:50 | 200 |      45.056µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:47:55 | 200 |      60.261µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:00 | 200 |      48.005µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:00 | 200 |      41.379µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:05 | 200 |      40.835µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:10 | 200 |      41.645µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:10 | 200 |      39.699µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:15 | 200 |      61.046µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:20 | 200 |      37.997µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:20 | 200 |      38.679µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:25 | 200 |      55.192µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:30 | 200 |      40.972µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:30 | 200 |      40.606µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:35 | 200 |      35.154µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:40 | 200 |      33.125µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:40 | 200 |      33.081µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:45 | 200 |      31.304µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:50 | 200 |      32.065µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:50 | 200 |      35.683µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:48:55 | 200 |       33.83µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:00 | 200 |      36.601µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:00 | 200 |      33.944µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:05 | 200 |      58.834µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:10 | 200 |      32.487µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:10 | 200 |      35.426µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:15 | 200 |      33.366µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:49:17.144Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"21bab23c-2a9c-4704-a511-63de27356333","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:49:17.144Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"21bab23c-2a9c-4704-a511-63de27356333","keyPrefix":"sk-oai-nSYIiErHJmVh...","username":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:49:17 | 201 |   10.976781ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:49:17 | 200 |    1.425451ms |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:49:17 | 200 |      149.03µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:49:20 | 200 |      32.465µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:20 | 200 |       34.19µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:25 | 200 |      59.094µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:30 | 200 |      35.047µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:30 | 200 |      40.084µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:35 | 200 |       33.35µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:40 | 200 |      32.824µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:40 | 200 |      957.04µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:49:40 | 200 |     138.868µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:49:40 | 200 |      31.726µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:40 | 404 |    1.091363ms |  44.212.242.249 | DELETE   "/v1/api-keys/e2e-trlp-test-key" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:49:45 | 200 |      32.978µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:50 | 200 |      35.316µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:50 | 200 |      35.963µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:49:55 | 200 |      37.149µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:00 | 200 |      35.002µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:00 | 200 |      31.939µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:05 | 200 |      34.645µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:10 | 200 |      34.026µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:10 | 200 |       31.13µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:15 | 200 |      33.547µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:20 | 200 |      34.798µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:20 | 200 |      33.671µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:25 | 200 |      33.783µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:30 | 200 |      33.249µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:30 | 200 |      35.044µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:50:31.251Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"b9cfa1de-90b7-45cd-a4c8-af999cba086e","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:50:31.251Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b9cfa1de-90b7-45cd-a4c8-af999cba086e","keyPrefix":"sk-oai-P6OEXqoB6LaL...","username":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:50:31 | 201 |     2.03924ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:50:31 | 200 |     609.133µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:50:31 | 200 |     129.827µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:50:35 | 200 |      37.029µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:40 | 200 |      40.259µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:40 | 200 |       39.35µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:45 | 200 |      47.016µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:50 | 200 |      43.258µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:50:50.481Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"6891d42c-9f04-4bc2-9c37-274a7252540a","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:50:50.481Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"6891d42c-9f04-4bc2-9c37-274a7252540a","keyPrefix":"sk-oai-1WOKhQx5nUL6...","username":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:50:50 | 201 |    2.358995ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:50:50 | 200 |     667.413µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:50:50 | 200 |      136.46µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:50:50 | 200 |     652.871µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:50:50 | 200 |     149.451µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:50:50 | 200 |      43.736µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:50:55 | 200 |      38.427µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:00 | 200 |      39.642µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:00 | 200 |      42.102µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:05 | 200 |      38.403µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:51:09.985Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"62304134-73e4-4eca-af3c-2871a3f3f6a3","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:51:09.985Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"62304134-73e4-4eca-af3c-2871a3f3f6a3","keyPrefix":"sk-oai-wtlscJOQfV3n...","username":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:51:09 | 201 |     2.03587ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:51:10 | 200 |     601.223µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:51:10 | 200 |     499.041µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:51:10 | 200 |     138.905µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:51:10 | 200 |    21.81231ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:51:10 | 200 |      39.029µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:10 | 200 |      39.608µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:15 | 200 |      38.578µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:20 | 200 |      41.588µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:20 | 200 |      41.195µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:25 | 200 |      44.533µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:29 | 200 |     187.802µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:51:29 | 200 |     174.534µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:51:29 | 200 |   40.733173ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:51:30 | 200 |      41.893µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:30 | 200 |      40.714µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:35 | 200 |      38.541µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:40 | 200 |      40.484µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:40 | 200 |      39.432µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:51:41.248Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3bbcbe9a-ee59-49af-91c6-d69d3bba1a99","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:51:41.248Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"3bbcbe9a-ee59-49af-91c6-d69d3bba1a99","keyPrefix":"sk-oai-1AluKcCznosy...","username":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:51:41 | 201 |    2.114017ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:51:45 | 200 |      39.096µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:50 | 200 |      37.712µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:50 | 200 |      44.422µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:51:55 | 200 |      54.768µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:00 | 200 |      40.272µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:00 | 200 |      36.122µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:05 | 200 |      39.626µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:10 | 200 |      38.764µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:10 | 200 |      39.156µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:11 | 200 |     969.121µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:11 | 200 |     133.777µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:52:11 | 200 |     502.992µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:11 | 200 |     505.308µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:11 | 200 |     125.301µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:52:11 | 200 |   13.764736ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:15 | 200 |      45.723µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:20 | 200 |      39.818µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:20 | 200 |      45.171µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:25 | 200 |      37.208µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:28 | 200 |      138.69µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:52:28 | 200 |   23.216578ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:30 | 200 |      40.678µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:30 | 200 |      45.221µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:35 | 200 |      37.398µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:52:37.125Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"0c218650-81c1-400d-9c61-ea169de206be","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:52:37.125Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"0c218650-81c1-400d-9c61-ea169de206be","keyPrefix":"sk-oai-yNaZCVZD74xI...","username":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:52:37 | 201 |    2.200978ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:40 | 200 |      40.346µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:40 | 200 |      36.302µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:45 | 200 |     722.594µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:45 | 200 |     428.914µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:45 | 200 |     125.408µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:52:45 | 200 |   12.460841ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:45 | 200 |      32.937µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:52:45.689Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"e1f15d8c-ed01-4e0e-b40f-5e695335ad90","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:52:45.689Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"e1f15d8c-ed01-4e0e-b40f-5e695335ad90","keyPrefix":"sk-oai-12rlZ83QlnS1...","username":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:52:45 | 201 |    1.744818ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:50 | 200 |      39.339µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:50 | 200 |      38.467µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:52:53 | 200 |     790.165µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:53 | 200 |     554.587µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:53 | 200 |     121.982µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:52:53 | 200 |    8.738699ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:53 | 200 |     451.862µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:53 | 200 |     435.063µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:52:53 | 200 |     105.212µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:52:53 | 200 |    8.515663ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-06-12T19:52:54.736Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"f9e2e006-29a0-4719-a07b-d204d0fbba40","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:52:54.736Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"f9e2e006-29a0-4719-a07b-d204d0fbba40","keyPrefix":"sk-oai-ZAmpsFOkEeFn...","username":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:52:54 | 201 |    1.931275ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:52:55 | 200 |      43.881µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:00 | 200 |      38.136µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:00 | 200 |      38.313µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:02 | 200 |     811.288µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:02 | 200 |     408.931µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:02 | 200 |     135.102µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:53:02 | 200 |   13.066136ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:53:05 | 200 |      37.906µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:10 | 200 |      38.613µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:10 | 200 |      39.638µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:53:11.994Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"4e41d6d7-5b7a-4e5c-b471-189cd76aa876","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:53:11.994Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"4e41d6d7-5b7a-4e5c-b471-189cd76aa876","keyPrefix":"sk-oai-YYDA6lUxsGij...","username":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:53:11 | 201 |    2.047415ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:53:15 | 200 |      39.241µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:20 | 200 |     752.851µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:20 | 200 |     444.136µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:20 | 200 |     155.734µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:53:20 | 200 |     489.086µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:20 | 200 |     128.113µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:53:20 | 200 |   23.437935ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:53:20 | 200 |       40.63µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:20 | 200 |      39.498µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:25 | 200 |       40.58µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:30 | 200 |      38.844µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:30 | 200 |      39.114µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:53:33.507Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d4186cc8-33de-4d1d-a05c-bb82c023494e","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:53:33.507Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d4186cc8-33de-4d1d-a05c-bb82c023494e","keyPrefix":"sk-oai-1A4X1eLf2l0Z...","username":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:53:33 | 201 |    3.413706ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:53:35 | 200 |      38.763µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:40 | 200 |      52.114µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:40 | 200 |      40.644µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:41 | 200 |     785.624µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:41 | 200 |     439.082µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:53:41 | 200 |     136.774µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:53:41 | 200 |    16.34791ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:53:45 | 200 |      45.316µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:50 | 200 |      38.225µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:50 | 200 |      41.549µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:55 | 200 |      40.595µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:53:55 | 200 |     173.503µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:53:55 | 200 |     175.798µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:53:55 | 200 |   45.722488ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:00 | 200 |      49.321µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:00 | 200 |      37.043µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:05 | 200 |      39.036µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:10 | 200 |       48.16µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:10 | 200 |      40.058µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:13 | 200 |     163.974µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:54:13 | 200 |   20.805395ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:15 | 200 |      39.264µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:20 | 200 |       39.24µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:20 | 200 |      39.262µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:54:24.390Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"47793669-a201-4db0-89ef-d87808d449ff","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:54:24.390Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"47793669-a201-4db0-89ef-d87808d449ff","keyPrefix":"sk-oai-1KxSkV67jXbC...","username":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:54:24 | 201 |    10.55217ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:25 | 200 |      40.381µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:30 | 200 |      43.334µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:30 | 200 |      40.353µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:32 | 200 |    1.648367ms |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:32 | 200 |     133.999µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-06-12T19:54:33.035Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"7bf93ea0-354c-43ff-8705-ba6ee3b87a8b","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:54:33.035Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"7bf93ea0-354c-43ff-8705-ba6ee3b87a8b","keyPrefix":"sk-oai-igjDOF9edLU3...","username":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:54:33 | 201 |    1.850077ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:35 | 200 |      41.001µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:40 | 200 |      43.651µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:40 | 200 |      38.719µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:41 | 200 |    1.041159ms |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:41 | 200 |     561.936µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:41 | 200 |     174.395µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:54:41 | 200 |   12.493164ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-06-12T19:54:41.454Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"8154a389-0571-4423-ae67-f86f1b2cbc2b","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:54:41.454Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"8154a389-0571-4423-ae67-f86f1b2cbc2b","keyPrefix":"sk-oai-4oH1S6xIHMQB...","username":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:54:41 | 201 |    1.816002ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:45 | 200 |      39.994µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:49 | 200 |     923.965µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:49 | 200 |     525.833µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:49 | 200 |      167.28µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:54:49 | 200 |   12.421701ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:50 | 200 |       41.23µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:54:50.243Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"d428cd42-aaac-4001-ab8d-1584319048db","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:54:50.243Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"d428cd42-aaac-4001-ab8d-1584319048db","keyPrefix":"sk-oai-1aCCASlvfBoq...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:54:50 | 201 |    1.887736ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:54:50 | 200 |      40.111µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:55 | 200 |      43.728µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:54:58 | 200 |     919.329µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:58 | 200 |     437.713µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:54:58 | 200 |     149.058µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:54:58 | 200 |   12.666555ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:55:00 | 200 |      46.729µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:00 | 200 |      45.465µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:05 | 200 |      39.509µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:55:07.480Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"b197852f-9862-4455-aae4-87ec0837b654","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:55:07.480Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"b197852f-9862-4455-aae4-87ec0837b654","keyPrefix":"sk-oai-1YOI9e7no9uh...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:55:07 | 201 |    2.061563ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:55:10 | 200 |      68.412µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:10 | 200 |      40.584µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:15 | 200 |      40.763µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:20 | 200 |      38.847µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:20 | 200 |      51.057µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:23 | 200 |     820.314µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
{"level":"error","timestamp":"2026-06-12T19:55:23.769Z","caller":"token/handler.go:65","message":"Missing or empty username header","header":"X-MaaS-Username","stacktrace":"main.registerHandlers.(*Handler).ExtractUserInfo.func1\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:65\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"}
[GIN] 2026/06/12 - 19:55:23 | 500 |       92.16µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:55:25 | 200 |      39.333µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:30 | 200 |      41.023µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:30 | 200 |      40.451µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:35 | 200 |      42.356µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:40 | 200 |       38.71µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:40 | 200 |      38.758µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:40 | 403 |     141.855µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:55:45 | 200 |       32.91µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:50 | 200 |      38.116µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:50 | 200 |      41.678µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:55 | 200 |      41.166µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:55:58 | 403 |     144.469µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:56:00 | 200 |      44.346µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:00 | 200 |      40.094µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:05 | 200 |      39.531µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:10 | 200 |      37.486µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:10 | 200 |      39.968µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:15 | 200 |      38.938µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:15 | 403 |     142.467µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:56:20 | 200 |      41.893µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:20 | 200 |      39.336µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:25 | 200 |      50.344µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:56:29.598Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-api-key-ignores-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"610da1f1-7af4-499f-92d4-cb0cf51b4ef4","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:56:29.598Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"610da1f1-7af4-499f-92d4-cb0cf51b4ef4","keyPrefix":"sk-oai-VxvHYTmbaY4c...","username":"system:serviceaccount:default:e2e-api-key-ignores-header-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:56:29 | 201 |    2.348541ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:56:30 | 200 |      58.441µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:30 | 200 |       42.23µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:35 | 200 |      40.426µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:37 | 200 |    1.075908ms |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:56:37 | 200 |      468.83µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:56:37 | 200 |     163.599µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:56:37 | 200 |   20.564574ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:56:40 | 200 |      37.715µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:40 | 200 |      40.699µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:45 | 200 |      44.814µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:50 | 200 |      41.843µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:50 | 200 |      61.081µs |      10.133.0.2 | GET      "/health"
{"level":"info","timestamp":"2026-06-12T19:56:51.861Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"2876bfe7-b3c8-401e-bb77-b00249e76285","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:56:51.861Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"2876bfe7-b3c8-401e-bb77-b00249e76285","keyPrefix":"sk-oai-1Jm2L7ZJqCfF...","username":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:56:51 | 201 |    3.708398ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
{"level":"info","timestamp":"2026-06-12T19:56:51.896Z","caller":"api_keys/service.go:150","message":"Created API key","user":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"fbe06d49-b75e-4eb6-a439-e96aa908cab7","ephemeral":false}
{"level":"info","timestamp":"2026-06-12T19:56:51.896Z","caller":"api_keys/handler.go:225","message":"Created API key","keyId":"fbe06d49-b75e-4eb6-a439-e96aa908cab7","keyPrefix":"sk-oai-1OuwBk51ZntM...","username":"system:serviceaccount:default:e2e-multi-keys-sa","groups":["system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false}
[GIN] 2026/06/12 - 19:56:51 | 201 |    1.858678ms |  44.212.242.249 | POST     "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:56:55 | 200 |      39.147µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:56:59 | 200 |     773.031µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:56:59 | 200 |     481.108µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:56:59 | 200 |     222.597µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:56:59 | 200 |   12.539313ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:56:59 | 200 |     508.835µs |     10.134.0.37 | POST     "/internal/v1/api-keys/validate"
[GIN] 2026/06/12 - 19:56:59 | 200 |     168.046µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:57:00 | 200 |       48.98µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:00 | 200 |      39.178µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:05 | 200 |      39.009µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:10 | 200 |      39.048µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:10 | 200 |       46.17µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:12 | 200 |     222.278µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:57:12 | 200 |     245.766µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:57:12 | 200 |   45.424204ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:57:15 | 200 |      39.827µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:20 | 200 |      48.094µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:20 | 200 |      40.241µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:25 | 200 |      38.325µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:26 | 200 |     185.023µs |     10.134.0.37 | POST     "/internal/v1/subscriptions/select"
[GIN] 2026/06/12 - 19:57:26 | 200 |   22.626276ms |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:57:26 | 200 |     144.996µs |  44.212.242.249 | GET      "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent
[GIN] 2026/06/12 - 19:57:30 | 200 |      42.688µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:30 | 200 |      46.081µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:35 | 200 |      38.581µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:40 | 200 |       37.96µs |      10.133.0.2 | GET      "/health"
[GIN] 2026/06/12 - 19:57:40 | 200 |      53.221µs |      10.133.0.2 | GET      "/health"