[GIN] 2026/07/06 - 20:39:16 | 200 | 277.687µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:16 | 200 | 681.213µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:16 | 200 | 240.851µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:16 | 200 | 647.863µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:16 | 200 | 140.855µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:17 | 200 | 566.919µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:17 | 200 | 261.256µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:39:17.237Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-rate-limit-test-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:39:17.237Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-rate-limit-test-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:39:17 | 200 | 1.691984ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:39:17 | 200 | 40.685µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:18 | 200 | 38.228µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:22 | 200 | 42.6µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:27 | 200 | 40.957µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:28 | 200 | 37.904µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:32 | 200 | 40.923µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:39:33.870Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"3e490c86-899a-436b-9e33-22ce7395717d","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:39:33.870Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"3e490c86-899a-436b-9e33-22ce7395717d","keyPrefix":"sk-oai-1eQVLk679tnU...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:39:33 | 201 | 2.241073ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:39:33 | 200 | 647.107µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:33 | 200 | 274.23µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:33 | 200 | 654.621µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:33 | 200 | 248.731µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:33 | 200 | 570.928µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:33 | 200 | 273.703µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:34 | 200 | 589.457µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:34 | 200 | 246.376µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:39:34.149Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-models-exempt-test-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:39:34.149Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-models-exempt-test-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:39:34 | 200 | 1.6341ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:39:37 | 200 | 39.004µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:38 | 200 | 38.699µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:38 | 200 | 978.972µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:38 | 200 | 305.218µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:39:38.758Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-extra-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:39:38 | 200 | 565.952µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:39:42 | 200 | 49.611µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:39:47.180Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"d5d278b6-455a-461c-89d6-66cc1a39f963","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:39:47.180Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"d5d278b6-455a-461c-89d6-66cc1a39f963","keyPrefix":"sk-oai-1wXeLFi0BSG7...","username":"system:serviceaccount:default:tester-regular-user","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:39:47 | 201 | 2.134196ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:39:47 | 200 | 600.046µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:47 | 200 | 244.246µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:47 | 200 | 40.597µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:48 | 200 | 38.149µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:49 | 200 | 1.276803ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:49 | 200 | 264.262µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:39:51 | 200 | 838.138µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:39:51 | 200 | 260.857µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:39:51.410Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-premium-sa-sub","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:39:51.410Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-premium-sa-sub","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:39:51 | 200 | 1.726636ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:39:52 | 200 | 52.11µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:57 | 200 | 38.284µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:39:58 | 200 | 38.198µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:02 | 200 | 42.145µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:04 | 200 | 1.047181ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:04 | 200 | 237.735µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:40:07 | 200 | 43.4µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:08 | 200 | 41.419µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:40:12.509Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-temp-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:40:12 | 200 | 896.073µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:40:12 | 200 | 584.562µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:12 | 200 | 188.762µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:40:12 | 200 | 40.354µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:40:17.264Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-second-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:40:17 | 200 | 895.044µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:40:17 | 200 | 38.267µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:18 | 200 | 43.482µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:21 | 200 | 927.781µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:21 | 200 | 262.247µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:40:21.660Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":14,"subscription":"simulator-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:40:21.660Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":14,"initiator":"controller"} [GIN] 2026/07/06 - 20:40:21 | 200 | 2.356805ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:40:22 | 200 | 38.68µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:27 | 200 | 68.928µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:28 | 200 | 36.006µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:40:30.393Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:40:30 | 200 | 894.067µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:40:30.420Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:40:30 | 200 | 482.436µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:40:30 | 200 | 656.771µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:32 | 200 | 42.576µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:37 | 200 | 71.248µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:38 | 200 | 39.513µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:41 | 200 | 2.244644ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:42 | 200 | 38.302µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:40:46.251Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-ordering-sa-657558","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"3177e304-91f5-460d-bba4-6b57bbfd070c","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:40:46.251Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"3177e304-91f5-460d-bba4-6b57bbfd070c","keyPrefix":"sk-oai-15JqAmjjUPvv...","username":"system:serviceaccount:models-as-a-service:e2e-ordering-sa-657558","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:40:46 | 201 | 2.127596ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:40:46 | 200 | 564.609µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:46 | 200 | 237.101µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:40:46 | 200 | 579.882µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:46 | 200 | 417.336µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:40:47 | 200 | 37.423µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:48 | 200 | 42.895µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:48 | 200 | 949.863µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:48 | 200 | 192.586µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:40:50 | 200 | 860.676µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:50 | 200 | 170.476µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:40:52 | 200 | 829.15µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:40:52 | 200 | 146.246µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:40:52.789Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-ordering-sub","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:40:52.789Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-ordering-sub","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:40:52 | 200 | 1.743297ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:40:52 | 200 | 41.097µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:57 | 200 | 61.353µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:40:58 | 200 | 39.691µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:02 | 200 | 40.905µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:07 | 200 | 38.776µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:08 | 200 | 38.872µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:12 | 200 | 39.24µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:16.508Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-success","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"9c524624-723d-4a64-b7e6-93403be05798","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:41:16.508Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"9c524624-723d-4a64-b7e6-93403be05798","keyPrefix":"sk-oai-1Ss8Q0J6C9Cr...","username":"system:serviceaccount:models-as-a-service:e2e-sa-success","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:41:16 | 201 | 2.408008ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:41:16 | 200 | 763.228µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:16 | 200 | 156.068µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:41:16.676Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-test-subscription-success","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:41:16.676Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-test-subscription-success","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:16 | 200 | 1.848565ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:17 | 200 | 44.35µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:18 | 200 | 38.753µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:21.614Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-no-sub","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"fbc218f4-8bb9-47e1-a6b4-6a35036bbd45","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:41:21.614Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"fbc218f4-8bb9-47e1-a6b4-6a35036bbd45","keyPrefix":"sk-oai-cAv7juiQD3ym...","username":"system:serviceaccount:models-as-a-service:e2e-sa-no-sub","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:41:21 | 201 | 1.870071ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:41:21.729Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"simulator-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:41:21.729Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:21 | 200 | 1.800835ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:41:21.756Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:21 | 200 | 515.541µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:22 | 200 | 39.389µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:25 | 200 | 808.46µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:27 | 200 | 38.354µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:28 | 200 | 38.633µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:31.333Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-sa-with-sub","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"56e7d30a-ba7e-4ef5-8260-96fc677a9f3c","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:41:31.333Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"56e7d30a-ba7e-4ef5-8260-96fc677a9f3c","keyPrefix":"sk-oai-6HgJ8jCuUEY8...","username":"system:serviceaccount:llm:e2e-sa-with-sub","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:41:31 | 201 | 2.051436ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:41:31 | 200 | 540.69µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:31 | 200 | 158.792µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:41:31.472Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-test-subscription-no-access","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:41:31.472Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-test-subscription-no-access","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:31 | 200 | 1.677017ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:32 | 200 | 41.917µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:36.493Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:36 | 200 | 880.784µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:37 | 200 | 39.347µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:38 | 200 | 39.538µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:40.940Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-single-sub","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"2e68145f-a8a2-4c6f-871d-d571c83c591a","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:41:40.940Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"2e68145f-a8a2-4c6f-871d-d571c83c591a","keyPrefix":"sk-oai-lzKT9qWek4t5...","username":"system:serviceaccount:models-as-a-service:e2e-sa-single-sub","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:41:40 | 201 | 1.931672ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:41:40 | 200 | 557.539µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:40 | 200 | 130.078µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:41:42 | 200 | 36.772µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:42 | 200 | 806.152µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:42 | 200 | 128.969µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:41:43.238Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-test-subscription-single-sub","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:41:43.238Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-test-subscription-single-sub","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:43 | 200 | 1.666987ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:47 | 200 | 38.87µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:48 | 200 | 51.002µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:52.397Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"079e09ad-8e46-4fe8-aa4a-bded7100caf0","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:41:52.397Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"079e09ad-8e46-4fe8-aa4a-bded7100caf0","keyPrefix":"sk-oai-hhzjttJ0lLGe...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:41:52 | 201 | 11.096655ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:41:52.433Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"405035fa-7463-4064-9c40-566a2f634263","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:41:52.433Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"405035fa-7463-4064-9c40-566a2f634263","keyPrefix":"sk-oai-EyGrBdQ4unnU...","username":"system:serviceaccount:models-as-a-service:e2e-sa-multi-sub-valid","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:41:52 | 201 | 1.942425ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:41:52 | 200 | 1.514308ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:52 | 200 | 184.255µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:41:52 | 200 | 712.549µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:41:52 | 200 | 182.279µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:41:52.624Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-test-subscription-free","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:41:52.624Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-test-subscription-free","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:52 | 200 | 2.007882ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:52 | 200 | 42.231µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:41:52.861Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-test-subscription-premium","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:41:52.861Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-test-subscription-premium","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:41:52 | 200 | 1.768505ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:41:57 | 200 | 50.532µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:41:58 | 200 | 37.994µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:02 | 200 | 41.262µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:07 | 200 | 42.169µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:08 | 200 | 41.753µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:12 | 200 | 41.924µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:17 | 200 | 39.739µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:18 | 200 | 44.74µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:22 | 200 | 47.761µs | 10.133.0.2 | GET "/health" {"level":"error","timestamp":"2026-07-06T20:42:26.649Z","caller":"token/handler.go:65","message":"Missing or empty username header","header":"X-MaaS-Username","stacktrace":"main.registerHandlers.(*Handler).ExtractUserInfo.func4\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:65\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/06 - 20:42:26 | 500 | 91.5µs | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:42:27 | 200 | 41.461µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:28 | 200 | 62.413µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:42:32.370Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"2e7338d5-05a8-4760-b220-2138f8aaf937","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:42:32.370Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"2e7338d5-05a8-4760-b220-2138f8aaf937","keyPrefix":"sk-oai-NNdERjFDYXiW...","username":"system:serviceaccount:models-as-a-service:e2e-sa-group-sub-only","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:42:32 | 201 | 2.465504ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:42:32 | 200 | 884.972µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:42:32 | 200 | 195.392µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:42:32.534Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-test-group-sub-only","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:42:32.534Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-test-group-sub-only","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:42:32 | 200 | 1.751597ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:42:32 | 200 | 39.136µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:37 | 200 | 40.772µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:42:37.971Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-status-active-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:42:37 | 200 | 995.316µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:42:38 | 200 | 50.681µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:42 | 200 | 44.227µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:42:43.088Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-status-failed-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:42:43 | 200 | 1.014501ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:42:47 | 200 | 41.383µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:48 | 200 | 43.623µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:52 | 200 | 49.876µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:42:57 | 200 | 41.982µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:42:57.908Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-status-degraded-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:42:57 | 200 | 1.111834ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:42:58 | 200 | 41.148µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:02 | 200 | 39.992µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:07 | 200 | 50.376µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:08 | 200 | 37.593µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:12 | 200 | 42.739µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:17 | 200 | 38.509µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:18 | 200 | 53.811µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:22 | 200 | 39.359µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:27 | 200 | 58.744µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:28 | 200 | 38.151µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:32 | 200 | 40.286µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:37 | 200 | 41.471µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:38 | 200 | 39.178µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:42 | 200 | 39.559µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:47 | 200 | 41.442µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:48 | 200 | 38.587µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:52 | 200 | 35.986µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:57 | 200 | 41.232µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:43:58 | 200 | 37.961µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:02 | 200 | 39.9µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:44:03.573Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"d60d57a2-356e-43b8-a716-1cff8dd0ecb5","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:44:03.573Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"d60d57a2-356e-43b8-a716-1cff8dd0ecb5","keyPrefix":"sk-oai-i3OmG2S87f5p...","username":"system:serviceaccount:llm:e2e-trlp-degraded-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:44:03 | 201 | 2.332792ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:44:03 | 200 | 638.142µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:44:03 | 200 | 147.162µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:44:07 | 200 | 48.777µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:08 | 200 | 42.139µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:12 | 200 | 39.303µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:17 | 200 | 39.809µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:18 | 200 | 49.099µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:22 | 200 | 65.782µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:27 | 200 | 50.219µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:28 | 200 | 39.277µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:30 | 200 | 2.164381ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:44:30 | 200 | 206.862µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:44:31 | 404 | 984.338µs | 34.228.250.194 | DELETE "/v1/api-keys/e2e-trlp-test-key" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:44:31.716Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-trlp-degraded-sub","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:44:31.716Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-trlp-degraded-sub","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:44:31 | 200 | 1.73641ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:44:31.838Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-trlp-degraded-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:44:31 | 200 | 618.959µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:44:32 | 200 | 39.655µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:37 | 200 | 47.901µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:38 | 200 | 46.93µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:42 | 200 | 39.166µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:47 | 200 | 37.436µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:48 | 200 | 40.404µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:52 | 200 | 43.501µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:44:53.032Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-status-transition-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:44:53 | 200 | 837.362µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:44:57 | 200 | 38.523µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:44:58 | 200 | 62.199µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:02 | 200 | 41.855µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:45:03.705Z","caller":"api_keys/service.go:473","message":"Ephemeral key cleanup completed","deletedCount":0} [GIN] 2026/07/06 - 20:45:03 | 200 | 3.14175ms | 10.133.0.67 | POST "/internal/v1/api-keys/cleanup" [GIN] 2026/07/06 - 20:45:07 | 200 | 45.787µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:45:08.243Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"27046c76-93b0-4def-b549-b768644fd597","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:45:08.243Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"27046c76-93b0-4def-b549-b768644fd597","keyPrefix":"sk-oai-AfCQ8jMjwvkl...","username":"system:serviceaccount:llm:e2e-degraded-healthy-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:45:08 | 201 | 2.224923ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:45:08 | 200 | 692.807µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:45:08 | 200 | 167.349µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:45:08.407Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-degraded-healthy-inf","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:45:08.407Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-degraded-healthy-inf","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:45:08 | 200 | 1.642986ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:45:08 | 200 | 38.257µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:12 | 200 | 47.941µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:17 | 200 | 40.976µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:18 | 200 | 59.723µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:22 | 200 | 39.212µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:45:23.513Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"4dbb558e-1e10-4e08-b9fd-edb00f6a6718","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:45:23.513Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"4dbb558e-1e10-4e08-b9fd-edb00f6a6718","keyPrefix":"sk-oai-1Cq845gG8ybO...","username":"system:serviceaccount:llm:e2e-failed-sub-inf-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:45:23 | 201 | 2.069744ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:45:23 | 200 | 560.429µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:45:23 | 200 | 162.873µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" {"level":"info","timestamp":"2026-07-06T20:45:27.022Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-failed-sub-inf","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:45:27.022Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-failed-sub-inf","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:45:27 | 200 | 2.050817ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:45:27 | 200 | 39.19µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:28 | 200 | 39.767µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:32 | 200 | 39.977µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:37 | 200 | 39.102µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:38 | 200 | 48.536µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:45:42.191Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"id":"d0682b93-47d7-4791-b063-f3f41e705716","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:45:42.191Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"d0682b93-47d7-4791-b063-f3f41e705716","keyPrefix":"sk-oai-gdpuGdpDW6HM...","username":"system:serviceaccount:llm:e2e-degraded-models-apikey-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:llm","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:45:42 | 201 | 2.041401ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:45:42 | 200 | 533.29µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:45:42 | 200 | 504.863µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:45:42 | 200 | 151.081µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:45:42 | 200 | 21.306018ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:45:42.364Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-degraded-models-apikey","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:45:42.364Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-degraded-models-apikey","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:45:42 | 200 | 1.570057ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:45:42 | 200 | 58.248µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:47 | 200 | 43.49µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:48 | 200 | 43.727µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:52 | 200 | 39.899µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:57 | 200 | 188.883µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:45:57 | 200 | 155.383µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:45:57 | 200 | 21.458473ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:45:57.643Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-degraded-models-kube","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:45:57 | 200 | 1.05554ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:45:57 | 200 | 40.943µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:45:58 | 200 | 41.578µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:02 | 200 | 54.008µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:46:02.930Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"simulator-subscription","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:46:02 | 200 | 788.761µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:46:07.800Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"927bcc69-3fd8-49f1-9677-3f1f411354aa","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:46:07.800Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"927bcc69-3fd8-49f1-9677-3f1f411354aa","keyPrefix":"sk-oai-QRD8XQZ2g5ia...","username":"system:serviceaccount:default:e2e-models-single-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:46:07 | 201 | 1.992618ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:07 | 200 | 47.229µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:08 | 200 | 41.38µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:11 | 200 | 846.465µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:11 | 200 | 467.582µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:11 | 200 | 200.556µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:11 | 200 | 26.239722ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:46:12.329Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-single-sub-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:46:12.329Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-single-sub-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:46:12 | 200 | 1.672762ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:46:12 | 200 | 41.213µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:17 | 200 | 37.929µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:18 | 200 | 44.149µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:20 | 200 | 154.06µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:20 | 200 | 17.890116ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:22 | 200 | 47.911µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:46:25.572Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"7e0c302a-dca5-4b08-8922-91302f472b9d","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:46:25.572Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"7e0c302a-dca5-4b08-8922-91302f472b9d","keyPrefix":"sk-oai-1czHqZULi83Z...","username":"system:serviceaccount:default:e2e-models-empty-header-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:46:25 | 201 | 2.054292ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:27 | 200 | 35.628µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:28 | 200 | 36.348µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:29 | 200 | 857.75µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:29 | 200 | 466.895µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:29 | 200 | 140.239µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:29 | 200 | 12.204564ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:46:30.120Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"af71f13a-b9ae-4700-a23a-0d369a80a743","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:46:30.120Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"af71f13a-b9ae-4700-a23a-0d369a80a743","keyPrefix":"sk-oai-108YRw96INUo...","username":"system:serviceaccount:default:e2e-models-filtered-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:46:30 | 201 | 1.66187ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:32 | 200 | 38.658µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:34 | 200 | 804.234µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:34 | 200 | 409.858µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:34 | 200 | 129.031µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:34 | 200 | 8.01262ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:34 | 200 | 570.236µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:34 | 200 | 445.272µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:34 | 200 | 153.57µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:34 | 200 | 8.006792ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:46:35.215Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"de06f29e-8b82-4fca-967e-9edd72a77a29","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:46:35.215Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"de06f29e-8b82-4fca-967e-9edd72a77a29","keyPrefix":"sk-oai-IsTNOypLcCB6...","username":"system:serviceaccount:default:e2e-models-dedup-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:46:35 | 201 | 1.910472ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:37 | 200 | 34.569µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:38 | 200 | 43.258µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:39 | 200 | 908.066µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" {"level":"error","timestamp":"2026-07-06T20:46:39.259Z","caller":"token/handler.go:65","message":"Missing or empty username header","header":"X-MaaS-Username","stacktrace":"main.registerHandlers.(*Handler).ExtractUserInfo.func1\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/token/handler.go:65\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.NewMiddleware.func8\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/metrics/middleware.go:20\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.LoggerWithConfig.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/logger.go:249\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\nmain.serve.RequestID.func7\n\tgithub.com/opendatahub-io/models-as-a-service/maas-api/internal/middleware/request_id.go:57\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.CustomRecoveryWithWriter.func1\n\tgithub.com/gin-gonic/gin@v1.10.1/recovery.go:102\ngithub.com/gin-gonic/gin.(*Context).Next\n\tgithub.com/gin-gonic/gin@v1.10.1/context.go:185\ngithub.com/gin-gonic/gin.(*Engine).handleHTTPRequest\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:644\ngithub.com/gin-gonic/gin.(*Engine).ServeHTTP\n\tgithub.com/gin-gonic/gin@v1.10.1/gin.go:600\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3340\nnet/http.(*conn).serve\n\tnet/http/server.go:2109"} [GIN] 2026/07/06 - 20:46:39 | 500 | 85.191µs | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:42 | 200 | 40.575µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:44 | 200 | 873.569µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:44 | 200 | 474.756µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:44 | 200 | 170.48µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:44 | 200 | 13.179513ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:46:44.423Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-dedup-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:46:44.423Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-dedup-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:46:44 | 200 | 1.90349ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:46:47 | 200 | 39µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:48 | 200 | 49.98µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:46:49.601Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"bb2285f8-102a-49a2-8eca-cd997014f3da","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:46:49.601Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"bb2285f8-102a-49a2-8eca-cd997014f3da","keyPrefix":"sk-oai-1596wEFoYemt...","username":"system:serviceaccount:default:e2e-models-diff-refs-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:46:49 | 201 | 2.006621ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:52 | 200 | 50.544µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:53 | 200 | 10.296324ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:53 | 200 | 506.355µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:53 | 200 | 171.241µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:53 | 200 | 8.109456ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:53 | 200 | 174.855µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:53 | 200 | 17.76787ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:46:57 | 200 | 43.31µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:58 | 200 | 38.441µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:46:58 | 200 | 1.017508ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:58 | 200 | 627.774µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:58 | 200 | 794.561µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:46:58 | 200 | 245.555µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:58 | 200 | 131.663µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:46:58 | 200 | 20.253205ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:46:58.844Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-diff-refs-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:46:58.844Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-diff-refs-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:46:58 | 200 | 2.144908ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:47:02 | 200 | 45.638µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:07 | 200 | 38.503µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:47:08.366Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"855efdda-3989-4d54-abe6-4f40bbe7fa97","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:47:08.366Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"855efdda-3989-4d54-abe6-4f40bbe7fa97","keyPrefix":"sk-oai-LxxHHk2Xik8O...","username":"system:serviceaccount:default:e2e-models-distinct-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:47:08 | 201 | 2.283348ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:47:08 | 200 | 40.203µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:12 | 200 | 1.023371ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:12 | 200 | 520.242µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:12 | 200 | 771.919µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:12 | 200 | 168.619µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:12 | 200 | 141.751µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:12 | 200 | 29.35517ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:47:12.556Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-distinct-models-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:47:12.556Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-distinct-models-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:12 | 200 | 2.303636ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:47:12.600Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-distinct-models-subscription","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:12 | 200 | 612.442µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:47:12 | 200 | 39.234µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:17 | 200 | 41.511µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:18 | 200 | 43.019µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:22 | 200 | 42.057µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:24 | 200 | 231.429µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:24 | 200 | 245.94µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:24 | 200 | 179.814µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:24 | 200 | 28.367418ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:47:24.882Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-return-all-sub1","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:24 | 200 | 1.051116ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:47:25.097Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-return-all-sub2","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:25 | 200 | 638.753µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:47:25.129Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-return-all-sub2","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:25 | 200 | 657.064µs | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:47:27 | 200 | 46.605µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:28 | 200 | 41.535µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:32 | 200 | 42.059µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:34 | 200 | 169.01µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:34 | 200 | 19.178733ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:47:34.408Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-user-token-filter-sub","tenant":"models-as-a-service","revokedCount":0,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:34 | 200 | 1.052635ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:47:37 | 200 | 41.714µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:38 | 200 | 38.98µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:42 | 200 | 39.829µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:47:45.664Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"55e861ff-6fc1-46d9-80ac-51b99c6bb209","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:47:45.664Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"55e861ff-6fc1-46d9-80ac-51b99c6bb209","keyPrefix":"sk-oai-iAWh1yPMeQxx...","username":"system:serviceaccount:default:e2e-empty-models-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:47:45 | 201 | 2.511975ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:47:47 | 200 | 40.23µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:48 | 200 | 37.086µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:49 | 200 | 999.582µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:49 | 200 | 231.524µs | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:47:49.826Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-empty-models-subscription","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:47:49.826Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-empty-models-subscription","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:47:49 | 200 | 1.735119ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" {"level":"info","timestamp":"2026-07-06T20:47:50.290Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"1bec544f-76a4-479e-b362-0dff3fc01069","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:47:50.290Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"1bec544f-76a4-479e-b362-0dff3fc01069","keyPrefix":"sk-oai-2tIya2uALRE3...","username":"system:serviceaccount:default:e2e-models-schema-test-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:47:50 | 201 | 1.786557ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:47:52 | 200 | 32.356µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:54 | 200 | 1.049276ms | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:54 | 200 | 441.486µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:54 | 200 | 128.641µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:54 | 200 | 12.591442ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:47:54.683Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"id":"8bda4241-8274-4d32-a99a-cbe8a90f8a5c","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:47:54.683Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"8bda4241-8274-4d32-a99a-cbe8a90f8a5c","keyPrefix":"sk-oai-uFBt6SXPWuhl...","username":"system:serviceaccount:default:e2e-models-metadata-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:default","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:47:54 | 201 | 1.942512ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:47:57 | 200 | 38.518µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:58 | 200 | 39.444µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:47:58 | 200 | 846.709µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:58 | 200 | 446.793µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:47:58 | 200 | 128.043µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:47:58 | 200 | 12.026186ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:47:59.438Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"2cd8210b-deb2-4b77-924a-4b8a3468b954","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:47:59.438Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"2cd8210b-deb2-4b77-924a-4b8a3468b954","keyPrefix":"sk-oai-Wfzxc39GtBJN...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-scoped-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:47:59 | 201 | 1.901776ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent [GIN] 2026/07/06 - 20:48:02 | 200 | 43.587µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:03 | 200 | 902.887µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:03 | 200 | 444.943µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:03 | 200 | 137.861µs | 10.134.0.31 | POST "/internal/v1/subscriptions/select" [GIN] 2026/07/06 - 20:48:03 | 200 | 14.870102ms | 34.228.250.194 | GET "/v1/models" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:48:03.605Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-api-key-scoped-sub","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:48:03.605Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-api-key-scoped-sub","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:48:03 | 200 | 1.738234ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:48:07 | 200 | 39.879µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:08 | 200 | 39.538µs | 10.133.0.2 | GET "/health" {"level":"info","timestamp":"2026-07-06T20:48:08.662Z","caller":"api_keys/service.go:194","message":"Created API key","user":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"id":"3037cd1f-6790-4cce-a497-1c29db4a15af","ephemeral":false} {"level":"info","timestamp":"2026-07-06T20:48:08.662Z","caller":"api_keys/handler.go:267","message":"Created API key","keyId":"3037cd1f-6790-4cce-a497-1c29db4a15af","keyPrefix":"sk-oai-68TDrvdUdMDC...","username":"system:serviceaccount:models-as-a-service:e2e-api-key-deleted-sub-sa","groups":["system:authenticated","system:serviceaccounts","system:serviceaccounts:models-as-a-service","system:authenticated"],"ephemeral":false} [GIN] 2026/07/06 - 20:48:08 | 201 | 2.056699ms | 34.228.250.194 | POST "/v1/api-keys" | Authorization=present X-Api-Key=absent Cookie=absent Set-Cookie=absent {"level":"info","timestamp":"2026-07-06T20:48:12.790Z","caller":"api_keys/store_postgres.go:575","message":"Revoked all API keys for subscription","count":1,"subscription":"e2e-api-key-deleted-sub","tenant":"models-as-a-service"} {"level":"info","timestamp":"2026-07-06T20:48:12.790Z","caller":"api_keys/handler.go:579","message":"Revoked API keys for subscription deletion","subscription":"e2e-api-key-deleted-sub","tenant":"models-as-a-service","revokedCount":1,"initiator":"controller"} [GIN] 2026/07/06 - 20:48:12 | 200 | 1.887496ms | 10.133.0.65 | POST "/internal/v1/api-keys/revoke-for-subscription" [GIN] 2026/07/06 - 20:48:12 | 200 | 40.14µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:16 | 200 | 874.684µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:17 | 200 | 39.913µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:18 | 200 | 39.361µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:21 | 200 | 861.69µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:22 | 200 | 38.701µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:26 | 200 | 866.395µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:27 | 200 | 46.63µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:28 | 200 | 38.262µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:32 | 200 | 863.627µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:32 | 200 | 46.188µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:37 | 200 | 828.726µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:37 | 200 | 44.555µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:38 | 200 | 39.444µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:42 | 200 | 818.72µs | 10.134.0.31 | POST "/internal/v1/api-keys/validate" [GIN] 2026/07/06 - 20:48:42 | 200 | 37.382µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:47 | 200 | 41.311µs | 10.133.0.2 | GET "/health" [GIN] 2026/07/06 - 20:48:48 | 200 | 47.945µs | 10.133.0.2 | GET "/health"