{"level":"info","ts":1789739818.718038,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","EoS":true} {"level":"info","ts":1789739818.7180638,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","header":"x-maas-username"} {"level":"info","ts":1789739818.7180743,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","header":"x-maas-subscription"} {"level":"info","ts":1789739818.7180798,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","header":"x-maas-keyname"} {"level":"info","ts":1789739818.7180848,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","header":"x-maas-group"} {"level":"info","ts":1789739818.718102,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789739818.7181582,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0"} {"level":"info","ts":1789739818.7189112,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0"} {"level":"info","ts":1789739818.7189348,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0"} {"level":"info","ts":1789739818.7189515,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","EoS":false} {"level":"info","ts":1789739818.7189717,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0"} {"level":"info","ts":1789739818.7189996,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0","EoS":true} {"level":"info","ts":1789739818.7190092,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e99009c4770872f78bc8902c58b04b3","span_id":"4f69a77e6fe4e409","x-request-id":"6b9e44a8-fd44-4196-acfd-65fda8f343a0"} {"level":"info","ts":1789739818.7598195,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789739818.7599225,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.759936,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.7599523,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","EoS":true} {"level":"info","ts":1789739818.7599812,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","header":"x-maas-keyname"} {"level":"info","ts":1789739818.7599914,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","header":"x-maas-username"} {"level":"info","ts":1789739818.759997,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","header":"x-maas-subscription"} {"level":"info","ts":1789739818.7600012,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","header":"x-maas-group"} {"level":"info","ts":1789739818.7600167,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789739818.7600474,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.7607987,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.760824,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.760841,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","EoS":false} {"level":"info","ts":1789739818.7608576,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.760892,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9","EoS":true} {"level":"info","ts":1789739818.7609065,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"087cadb29d104aeed35469fff677eb9f","span_id":"2f4bcdf8a366ec41","x-request-id":"8cbd989d-4e2a-4043-8e52-8a7ce4be8ae9"} {"level":"info","ts":1789739818.8000658,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789739818.8002007,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"info","ts":1789739818.8002148,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"info","ts":1789739818.8002284,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","EoS":true} {"level":"info","ts":1789739818.80025,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","header":"x-maas-username"} {"level":"info","ts":1789739818.800261,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","header":"x-maas-keyname"} {"level":"info","ts":1789739818.800266,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","header":"x-maas-subscription"} {"level":"info","ts":1789739818.8002713,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","header":"x-maas-group"} {"level":"info","ts":1789739818.8002875,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789739818.80032,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"info","ts":1789739818.801161,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"info","ts":1789739818.8011844,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"info","ts":1789739818.8012075,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","EoS":false} {"level":"info","ts":1789739818.8012254,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"info","ts":1789739818.8012607,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0","EoS":true} {"level":"info","ts":1789739818.8012776,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"306a0c04a616c4a5d87caa75fd080126","span_id":"288af6e2ac209d7a","x-request-id":"2bbd9f05-4b6d-49b0-b15f-d96ce6d504d0"} {"level":"error","ts":1789739831.036943,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789739832.6125798,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789739832.612691,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"292088f59ae4ae61a79b4dc01929e4cc","span_id":"c6e1d0eebbb8336d","x-request-id":"16f1d707-add9-4ce3-8d68-a6e0207774b1"} {"level":"info","ts":1789739832.6127083,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"292088f59ae4ae61a79b4dc01929e4cc","span_id":"c6e1d0eebbb8336d","x-request-id":"16f1d707-add9-4ce3-8d68-a6e0207774b1"} {"level":"info","ts":1789739832.6127255,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"292088f59ae4ae61a79b4dc01929e4cc","span_id":"c6e1d0eebbb8336d","x-request-id":"16f1d707-add9-4ce3-8d68-a6e0207774b1","EoS":true} {"level":"info","ts":1789739832.6127927,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"292088f59ae4ae61a79b4dc01929e4cc","span_id":"c6e1d0eebbb8336d","x-request-id":"16f1d707-add9-4ce3-8d68-a6e0207774b1"} {"level":"info","ts":1789739832.7689886,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789739832.7690701,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6d9c4f7c865b9298910c1c554fb61971","span_id":"4b10f28ac24155d2","x-request-id":"3b205344-0d86-4de0-b49b-d555e63689bb"} {"level":"info","ts":1789739832.7690825,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6d9c4f7c865b9298910c1c554fb61971","span_id":"4b10f28ac24155d2","x-request-id":"3b205344-0d86-4de0-b49b-d555e63689bb"} {"level":"info","ts":1789739832.7690935,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6d9c4f7c865b9298910c1c554fb61971","span_id":"4b10f28ac24155d2","x-request-id":"3b205344-0d86-4de0-b49b-d555e63689bb","EoS":true} {"level":"info","ts":1789739832.7691498,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6d9c4f7c865b9298910c1c554fb61971","span_id":"4b10f28ac24155d2","x-request-id":"3b205344-0d86-4de0-b49b-d555e63689bb"} {"level":"info","ts":1789739832.9182343,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789739832.9183166,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"162453930f337e20de0f5c17ca0fbe87","span_id":"08a7d438bb7868e2","x-request-id":"931c2ea0-3bd5-4fab-a3c3-0b4b4980e98d"} {"level":"info","ts":1789739832.918328,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"162453930f337e20de0f5c17ca0fbe87","span_id":"08a7d438bb7868e2","x-request-id":"931c2ea0-3bd5-4fab-a3c3-0b4b4980e98d"} {"level":"info","ts":1789739832.918339,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"162453930f337e20de0f5c17ca0fbe87","span_id":"08a7d438bb7868e2","x-request-id":"931c2ea0-3bd5-4fab-a3c3-0b4b4980e98d","EoS":true} {"level":"info","ts":1789739832.9183793,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"162453930f337e20de0f5c17ca0fbe87","span_id":"08a7d438bb7868e2","x-request-id":"931c2ea0-3bd5-4fab-a3c3-0b4b4980e98d"} {"level":"error","ts":1789739846.0390794,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789740811.410122,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789740811.4102645,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.4102776,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.41029,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","EoS":true} {"level":"info","ts":1789740811.4103167,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","header":"x-maas-group"} {"level":"info","ts":1789740811.4103312,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","header":"x-maas-username"} {"level":"info","ts":1789740811.4103363,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","header":"x-maas-keyname"} {"level":"info","ts":1789740811.4103417,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","header":"x-maas-subscription"} {"level":"info","ts":1789740811.4103572,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789740811.410382,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.411255,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.411278,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.4112937,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","EoS":false} {"level":"info","ts":1789740811.4113054,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.4113271,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5","EoS":true} {"level":"info","ts":1789740811.4113371,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"25d2e009f71beafee0c41daea30b7d13","span_id":"bb13b5ac0541bcd5","x-request-id":"4fa28064-610e-4ccd-90ec-c326c40e92b5"} {"level":"info","ts":1789740811.451748,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789740811.4518523,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"info","ts":1789740811.4518688,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"info","ts":1789740811.4518816,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","EoS":true} {"level":"info","ts":1789740811.4519055,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","header":"x-maas-subscription"} {"level":"info","ts":1789740811.451915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","header":"x-maas-keyname"} {"level":"info","ts":1789740811.4519212,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","header":"x-maas-group"} {"level":"info","ts":1789740811.451926,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","header":"x-maas-username"} {"level":"info","ts":1789740811.4519424,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789740811.4519703,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"info","ts":1789740811.4526932,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"info","ts":1789740811.4527128,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"info","ts":1789740811.4527254,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","EoS":false} {"level":"info","ts":1789740811.452737,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"info","ts":1789740811.4527588,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9","EoS":true} {"level":"info","ts":1789740811.4527683,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0e3bcfc2fa6e75a98cf0e726b46cc18b","span_id":"7a6ce56f018ddb31","x-request-id":"9c47bb2f-6d78-43e9-bb6c-c4725cf280c9"} {"level":"error","ts":1789740826.1500175,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741144.34536,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5f545ff9-23fe-494b-88e3-48c9e3cc0a72"} {"level":"info","ts":1789741144.3453705,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"96416b49-c014-4173-ab05-3cd78c268c2e","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789741144.345505,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"96416b49-c014-4173-ab05-3cd78c268c2e","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.34554,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"96416b49-c014-4173-ab05-3cd78c268c2e"} {"level":"error","ts":1789741144.35337,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5f545ff9-23fe-494b-88e3-48c9e3cc0a72","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.3533936,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0e9a0f58-e271-4da8-8214-b09e73cba271","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789741144.3534222,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0e9a0f58-e271-4da8-8214-b09e73cba271","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.3534477,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0e9a0f58-e271-4da8-8214-b09e73cba271"} {"level":"info","ts":1789741144.3602731,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1103cdf6-be54-4a2c-b448-4c0e92ff691a","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789741144.3603003,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1103cdf6-be54-4a2c-b448-4c0e92ff691a","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.3603292,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1103cdf6-be54-4a2c-b448-4c0e92ff691a"} {"level":"error","ts":1789741144.3603532,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5f545ff9-23fe-494b-88e3-48c9e3cc0a72","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.3604183,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"cb779b42-5e7d-45ed-b2d7-85964b229f49"} {"level":"error","ts":1789741144.3604467,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"cb779b42-5e7d-45ed-b2d7-85964b229f49","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.3650808,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"cb779b42-5e7d-45ed-b2d7-85964b229f49","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.3752568,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f9534e1a-5918-41e1-a690-d0f2fd58b6b1"} {"level":"error","ts":1789741144.375286,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f9534e1a-5918-41e1-a690-d0f2fd58b6b1","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.3795135,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f9534e1a-5918-41e1-a690-d0f2fd58b6b1","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.399662,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8af99b48-08ba-4745-9bc3-cdbda9b931c6"} {"level":"error","ts":1789741144.3997009,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8af99b48-08ba-4745-9bc3-cdbda9b931c6","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.4042416,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8af99b48-08ba-4745-9bc3-cdbda9b931c6","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.4443653,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0a8ed3b3-63b2-4c5d-b76a-1959279962ad"} {"level":"error","ts":1789741144.4444025,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0a8ed3b3-63b2-4c5d-b76a-1959279962ad","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.4487393,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0a8ed3b3-63b2-4c5d-b76a-1959279962ad","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.479697,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"b7c1d16c-7c5f-4e55-8728-82395cd5883f"} {"level":"info","ts":1789741144.4797213,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"24acad38-70e5-461d-a481-c60f9aa9b83c","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789741144.4797432,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"b7c1d16c-7c5f-4e55-8728-82395cd5883f","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.4797518,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"24acad38-70e5-461d-a481-c60f9aa9b83c","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.4797828,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"24acad38-70e5-461d-a481-c60f9aa9b83c"} {"level":"error","ts":1789741144.4889722,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"b7c1d16c-7c5f-4e55-8728-82395cd5883f","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.6495037,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1b6d2655-986d-46f4-be35-18c9a9b5c35e"} {"level":"error","ts":1789741144.6495478,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1b6d2655-986d-46f4-be35-18c9a9b5c35e","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.6550078,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1b6d2655-986d-46f4-be35-18c9a9b5c35e","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741144.9752622,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3f8e8d07-9176-4a84-a2f3-01cc71544dc9"} {"level":"error","ts":1789741144.9753165,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3f8e8d07-9176-4a84-a2f3-01cc71544dc9","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741144.9801216,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3f8e8d07-9176-4a84-a2f3-01cc71544dc9","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741145.621294,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"b6fb7f8b-d763-409f-9a69-cf127ffe6ec3"} {"level":"error","ts":1789741145.621337,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"b6fb7f8b-d763-409f-9a69-cf127ffe6ec3","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741145.626721,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"b6fb7f8b-d763-409f-9a69-cf127ffe6ec3","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741146.9070978,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c195cbf7-e409-417a-9e7e-7356c996c1c1"} {"level":"error","ts":1789741146.9071934,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c195cbf7-e409-417a-9e7e-7356c996c1c1","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741146.913278,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c195cbf7-e409-417a-9e7e-7356c996c1c1","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741149.4737973,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a801e24b-0432-40ef-8536-1528d48b047d"} {"level":"error","ts":1789741149.4738717,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a801e24b-0432-40ef-8536-1528d48b047d","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789741149.4789522,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a801e24b-0432-40ef-8536-1528d48b047d","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741149.4798987,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2dd6cbd8-b408-4208-aa2a-6a0d503716cb","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789741149.4799452,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2dd6cbd8-b408-4208-aa2a-6a0d503716cb","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789741149.4799829,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"2dd6cbd8-b408-4208-aa2a-6a0d503716cb"} {"level":"info","ts":1789741149.7748485,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741149.774961,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.7749739,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.7749867,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","EoS":true} {"level":"info","ts":1789741149.7750096,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","header":"x-maas-username"} {"level":"info","ts":1789741149.7750204,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","header":"x-maas-group"} {"level":"info","ts":1789741149.7750256,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","header":"x-maas-keyname"} {"level":"info","ts":1789741149.7750309,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","header":"x-maas-subscription"} {"level":"info","ts":1789741149.7750456,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741149.7750778,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.7759182,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.7759378,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.7759516,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","EoS":false} {"level":"info","ts":1789741149.7759628,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.7759893,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615","EoS":true} {"level":"info","ts":1789741149.776002,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f2e6710fbe58799411bfd1bb4bd76ac8","span_id":"93d5028b16b5e972","x-request-id":"05109b6b-f7f9-431e-a9a5-088328ec8615"} {"level":"info","ts":1789741149.8871253,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741149.887245,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.8872583,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.8872712,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","EoS":true} {"level":"info","ts":1789741149.8872921,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","header":"x-maas-username"} {"level":"info","ts":1789741149.8873024,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","header":"x-maas-subscription"} {"level":"info","ts":1789741149.8873072,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","header":"x-maas-group"} {"level":"info","ts":1789741149.887312,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","header":"x-maas-keyname"} {"level":"info","ts":1789741149.8873265,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741149.8873496,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.8880184,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.8880322,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.888041,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","EoS":false} {"level":"info","ts":1789741149.8880515,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.8880785,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb","EoS":true} {"level":"info","ts":1789741149.88809,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c4a8cbe019521879e848bdba4d0e1ced","span_id":"b1a2766e7dc044f5","x-request-id":"53d1bf21-50e6-471b-9a8e-53b9aa6998bb"} {"level":"info","ts":1789741149.9775429,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741149.977635,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741149.9776487,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741149.9776604,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","EoS":true} {"level":"info","ts":1789741149.9776802,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","header":"x-maas-keyname"} {"level":"info","ts":1789741149.9776895,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","header":"x-maas-group"} {"level":"info","ts":1789741149.9776957,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","header":"x-maas-subscription"} {"level":"info","ts":1789741149.9777002,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","header":"x-maas-username"} {"level":"info","ts":1789741149.9777145,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741149.9777393,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741149.97845,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741149.9784646,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741149.9784737,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","EoS":false} {"level":"info","ts":1789741149.9784825,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741149.9785075,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc","EoS":true} {"level":"info","ts":1789741149.9785166,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dcc96319d302ea1d93dfed9c18025948","span_id":"d70e4d50034f7d40","x-request-id":"414bbb54-e804-4b3e-999f-d34fcc6919cc"} {"level":"info","ts":1789741150.1128721,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0cfe1ef0-9588-4e72-93ba-4763392267f4","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789741150.1129258,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"0cfe1ef0-9588-4e72-93ba-4763392267f4","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789741150.1128721,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"394eadc9-c0ac-446d-8b83-e09ce369f478"} {"level":"error","ts":1789741161.1951177,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741178.3885577,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741178.3886518,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.3886638,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.3886757,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","EoS":true} {"level":"info","ts":1789741178.3887022,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","header":"x-maas-username"} {"level":"info","ts":1789741178.3887124,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","header":"x-maas-keyname"} {"level":"info","ts":1789741178.3887188,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","header":"x-maas-subscription"} {"level":"info","ts":1789741178.3887236,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","header":"x-maas-group"} {"level":"info","ts":1789741178.3887374,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741178.3887625,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.4016066,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.4016356,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.4016528,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","EoS":false} {"level":"info","ts":1789741178.401667,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.4016929,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934","EoS":true} {"level":"info","ts":1789741178.401701,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fa174906243771b96e3de5df71bfdb84","span_id":"d4f566fec4e566a9","x-request-id":"1cca70be-cc0f-4395-9701-2358ab87a934"} {"level":"info","ts":1789741178.538497,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741178.5385864,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.538599,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.538611,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","EoS":true} {"level":"info","ts":1789741178.5386343,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","header":"x-maas-subscription"} {"level":"info","ts":1789741178.538644,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","header":"x-maas-username"} {"level":"info","ts":1789741178.5386503,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","header":"x-maas-keyname"} {"level":"info","ts":1789741178.538656,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","header":"x-maas-group"} {"level":"info","ts":1789741178.5386696,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741178.5386953,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.5408988,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.5409217,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.540941,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","EoS":false} {"level":"info","ts":1789741178.5409596,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.5409946,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce","EoS":true} {"level":"info","ts":1789741178.5410085,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ef5350f20cc438ce03816c6eda289dc","span_id":"4aefbde91fa872c0","x-request-id":"7a8c1690-b72f-4f4b-87ce-9a9bc79b96ce"} {"level":"info","ts":1789741178.6772017,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741178.677305,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.677319,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.6773312,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","EoS":true} {"level":"info","ts":1789741178.6773567,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","header":"x-maas-group"} {"level":"info","ts":1789741178.6773672,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","header":"x-maas-username"} {"level":"info","ts":1789741178.677373,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","header":"x-maas-keyname"} {"level":"info","ts":1789741178.6773777,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","header":"x-maas-subscription"} {"level":"info","ts":1789741178.6773908,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741178.6774144,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.6782815,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.6782959,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.6783073,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","EoS":false} {"level":"info","ts":1789741178.6783168,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.6783392,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320","EoS":true} {"level":"info","ts":1789741178.6783476,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16c6fe242b3be53046362f1930dc1696","span_id":"b2bc086307abb792","x-request-id":"0e9ea2f8-fa06-422d-bfec-ae41433c4320"} {"level":"info","ts":1789741178.812524,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741178.8126225,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.8126383,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.8126512,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","EoS":true} {"level":"info","ts":1789741178.8126714,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","header":"x-maas-subscription"} {"level":"info","ts":1789741178.812681,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","header":"x-maas-group"} {"level":"info","ts":1789741178.812687,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","header":"x-maas-keyname"} {"level":"info","ts":1789741178.8126915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","header":"x-maas-username"} {"level":"info","ts":1789741178.812706,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741178.81273,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.8135734,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.8135877,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.8136003,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","EoS":false} {"level":"info","ts":1789741178.8136103,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.8136346,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74","EoS":true} {"level":"info","ts":1789741178.8136463,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b49f85da0993bbdec7e9bb3825905164","span_id":"091ed8e52e3144ba","x-request-id":"cec6eba1-1298-45ab-9f40-17253f7fba74"} {"level":"info","ts":1789741178.9484398,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741178.94855,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741178.948563,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741178.9485755,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","EoS":true} {"level":"info","ts":1789741178.9485996,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","header":"x-maas-username"} {"level":"info","ts":1789741178.94861,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","header":"x-maas-keyname"} {"level":"info","ts":1789741178.9486163,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","header":"x-maas-subscription"} {"level":"info","ts":1789741178.9486206,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","header":"x-maas-group"} {"level":"info","ts":1789741178.9486344,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741178.948661,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741178.9495206,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741178.9495392,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741178.9495513,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","EoS":false} {"level":"info","ts":1789741178.9495618,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741178.949583,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794","EoS":true} {"level":"info","ts":1789741178.9495928,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2a930c9c70b645d090b755561895441b","span_id":"38f4f2d892ea3275","x-request-id":"34488ea1-ab66-43a2-b116-6ccbd1b7f794"} {"level":"info","ts":1789741187.7216406,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741187.7217424,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.721757,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.72177,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","EoS":true} {"level":"info","ts":1789741187.721799,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","header":"x-maas-subscription"} {"level":"info","ts":1789741187.7218091,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","header":"x-maas-keyname"} {"level":"info","ts":1789741187.721814,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","header":"x-maas-username"} {"level":"info","ts":1789741187.7218187,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","header":"x-maas-group"} {"level":"info","ts":1789741187.7218332,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741187.7218628,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.7227702,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.722794,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.7228158,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","EoS":false} {"level":"info","ts":1789741187.7228367,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.7228642,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d","EoS":true} {"level":"info","ts":1789741187.7228727,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"76553c54d18e59e6794cc00df5dfd56f","span_id":"e842d5e4ebfec211","x-request-id":"e3a3c06a-6db6-48d7-9701-dff0ae00985d"} {"level":"info","ts":1789741187.8216348,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741187.8217216,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66"} {"level":"info","ts":1789741187.8225486,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66"} {"level":"info","ts":1789741187.8225632,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66"} {"level":"info","ts":1789741187.8225758,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66","EoS":false} {"level":"info","ts":1789741187.8225875,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66"} {"level":"info","ts":1789741187.8226194,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66","EoS":true} {"level":"info","ts":1789741187.8226275,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ccfae381082ecbf0c1fb1fce0d9f17fd","span_id":"bfcfa662c84e6402","x-request-id":"9c3ad38e-4844-4981-88a5-4f387424cf66"} {"level":"info","ts":1789741188.5522485,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741188.5523763,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741188.552391,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741188.552406,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","EoS":true} {"level":"info","ts":1789741188.55243,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","header":"x-maas-group"} {"level":"info","ts":1789741188.5524395,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","header":"x-maas-keyname"} {"level":"info","ts":1789741188.5524457,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","header":"x-maas-username"} {"level":"info","ts":1789741188.5524507,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","header":"x-maas-subscription"} {"level":"info","ts":1789741188.5524654,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741188.55249,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741188.5531914,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741188.5532112,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741188.5532262,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","EoS":false} {"level":"info","ts":1789741188.5532467,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741188.5532768,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f","EoS":true} {"level":"info","ts":1789741188.5532863,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3fc62470b91534fdd78e10425998acbb","span_id":"ed26deeba9f234f2","x-request-id":"17716191-e7d3-42a1-b29b-1e3876bb3a2f"} {"level":"info","ts":1789741189.4686062,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741189.4686997,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741189.4687116,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741189.4687238,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","EoS":true} {"level":"info","ts":1789741189.4687464,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","header":"x-maas-subscription"} {"level":"info","ts":1789741189.4687562,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","header":"x-maas-group"} {"level":"info","ts":1789741189.4687605,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","header":"x-maas-username"} {"level":"info","ts":1789741189.468765,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","header":"x-maas-keyname"} {"level":"info","ts":1789741189.4687796,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741189.4688067,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741189.469549,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741189.4695685,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741189.4695826,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","EoS":false} {"level":"info","ts":1789741189.469594,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741189.4696143,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb","EoS":true} {"level":"info","ts":1789741189.4696214,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c9d27a53210f1eab47249abe57623889","span_id":"67d201e653e596ef","x-request-id":"826f3fac-4af7-4fef-b03f-19e4f647bbdb"} {"level":"info","ts":1789741190.1646867,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741190.1647787,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741190.1647916,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741190.164804,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","EoS":true} {"level":"info","ts":1789741190.1648283,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","header":"x-maas-subscription"} {"level":"info","ts":1789741190.164839,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","header":"x-maas-keyname"} {"level":"info","ts":1789741190.164844,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","header":"x-maas-username"} {"level":"info","ts":1789741190.164848,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","header":"x-maas-group"} {"level":"info","ts":1789741190.1648614,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741190.1648867,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741190.1656187,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741190.1656413,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741190.1656559,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","EoS":false} {"level":"info","ts":1789741190.165673,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741190.165699,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2","EoS":true} {"level":"info","ts":1789741190.1657095,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"79d6b9543226aa88386178224cc9d3f3","span_id":"734eea611f979775","x-request-id":"466f4b9c-874a-42a4-8fd3-584c7653dff2"} {"level":"info","ts":1789741191.1779938,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741191.1780915,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"info","ts":1789741191.1781042,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"info","ts":1789741191.1781163,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","EoS":true} {"level":"info","ts":1789741191.1781633,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","header":"x-maas-group"} {"level":"info","ts":1789741191.178174,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","header":"x-maas-subscription"} {"level":"info","ts":1789741191.1781793,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","header":"x-maas-keyname"} {"level":"info","ts":1789741191.178184,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","header":"x-maas-username"} {"level":"info","ts":1789741191.178198,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741191.1782215,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"info","ts":1789741191.1789198,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"info","ts":1789741191.1789372,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"info","ts":1789741191.1789513,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","EoS":false} {"level":"info","ts":1789741191.1789677,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"info","ts":1789741191.178995,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68","EoS":true} {"level":"info","ts":1789741191.1790092,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c5cec53bc3ba4cf93d3d4db37db7141a","span_id":"6aa43f3abb409e3a","x-request-id":"0513e9fb-c40a-4565-b0ea-99636a31dd68"} {"level":"error","ts":1789741191.1977527,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789741201.1987028,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741210.6005685,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741210.6006544,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"info","ts":1789741210.6006684,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"info","ts":1789741210.6006796,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","EoS":true} {"level":"info","ts":1789741210.6007013,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","header":"x-maas-keyname"} {"level":"info","ts":1789741210.6007106,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","header":"x-maas-subscription"} {"level":"info","ts":1789741210.6007152,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","header":"x-maas-username"} {"level":"info","ts":1789741210.6007202,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","header":"x-maas-group"} {"level":"info","ts":1789741210.6007338,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741210.6007564,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"info","ts":1789741210.6014426,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"info","ts":1789741210.601468,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"info","ts":1789741210.6014893,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","EoS":false} {"level":"info","ts":1789741210.601506,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"info","ts":1789741210.6015394,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c","EoS":true} {"level":"info","ts":1789741210.6015472,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6ace428452dd2dba0fb30f2211cf0bc3","span_id":"3b47d4af60d0d489","x-request-id":"1e27f685-c5d8-4945-91e3-53a904c6381c"} {"level":"error","ts":1789741211.1992273,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789741221.1996908,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789741231.2003238,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741299.2289336,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741299.2290337,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"info","ts":1789741299.2290475,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"info","ts":1789741299.2290602,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","EoS":true} {"level":"info","ts":1789741299.2290804,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","header":"x-maas-username"} {"level":"info","ts":1789741299.229095,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","header":"x-maas-keyname"} {"level":"info","ts":1789741299.2291017,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","header":"x-maas-subscription"} {"level":"info","ts":1789741299.2291062,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","header":"x-maas-group"} {"level":"info","ts":1789741299.22912,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789741299.2291646,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"info","ts":1789741299.2417524,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"info","ts":1789741299.2417808,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"info","ts":1789741299.2418025,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","EoS":false} {"level":"info","ts":1789741299.2418206,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"info","ts":1789741299.2418623,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326","EoS":true} {"level":"info","ts":1789741299.2418797,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1cdf281b9a4dac71abdaff7df0443bff","span_id":"fe76184276066562","x-request-id":"6fb7fc91-17a3-456f-a515-72add7fd7326"} {"level":"error","ts":1789741311.2102842,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741379.2173896,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741379.217511,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.2175252,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.2175384,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","EoS":true} {"level":"info","ts":1789741379.21756,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","header":"x-maas-keyname"} {"level":"info","ts":1789741379.21757,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","header":"x-maas-username"} {"level":"info","ts":1789741379.2175772,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","header":"x-maas-subscription"} {"level":"info","ts":1789741379.2175822,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","header":"x-maas-group"} {"level":"info","ts":1789741379.217598,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741379.217628,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.218618,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.2186391,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.2186565,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","EoS":false} {"level":"info","ts":1789741379.2186685,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.2186913,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210","EoS":true} {"level":"info","ts":1789741379.2187047,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"882ea603ae1170de172ef090c91f6072","span_id":"7f7db64c26268c0a","x-request-id":"4ab7e87d-ed05-426f-b248-b04a25db3210"} {"level":"info","ts":1789741379.2560554,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741379.2561767,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"info","ts":1789741379.2561896,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"info","ts":1789741379.2562022,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","EoS":true} {"level":"info","ts":1789741379.256225,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","header":"x-maas-subscription"} {"level":"info","ts":1789741379.2562351,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","header":"x-maas-username"} {"level":"info","ts":1789741379.2562425,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","header":"x-maas-keyname"} {"level":"info","ts":1789741379.2562478,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","header":"x-maas-group"} {"level":"info","ts":1789741379.2562616,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789741379.2562861,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"info","ts":1789741379.2572043,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"info","ts":1789741379.2572315,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"info","ts":1789741379.257254,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","EoS":false} {"level":"info","ts":1789741379.257275,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"info","ts":1789741379.2573037,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52","EoS":true} {"level":"info","ts":1789741379.2573137,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9b113eda901640ee77b2299e2e0357fd","span_id":"36f9ff69aaa698fd","x-request-id":"4d827fa6-9b97-4450-bf0e-2950acb8ea52"} {"level":"error","ts":1789741391.2190228,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741392.9390626,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741392.9392118,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741392.9392347,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741392.9392545,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","EoS":true} {"level":"info","ts":1789741392.9392762,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","header":"x-maas-username"} {"level":"info","ts":1789741392.9392881,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","header":"x-maas-subscription"} {"level":"info","ts":1789741392.939296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","header":"x-maas-keyname"} {"level":"info","ts":1789741392.9393034,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","header":"x-maas-group"} {"level":"info","ts":1789741392.9393208,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789741392.939357,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741392.9521618,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741392.9521952,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741392.952222,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","EoS":false} {"level":"info","ts":1789741392.9522407,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741392.9522827,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec","EoS":true} {"level":"info","ts":1789741392.952298,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bf69b93a48f597dfc677eeb6a721e3c3","span_id":"c34316804aa28d30","x-request-id":"839ad733-362d-4594-ad55-34ffcc9662ec"} {"level":"info","ts":1789741393.0896084,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741393.0897093,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.089723,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.0897355,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","EoS":true} {"level":"info","ts":1789741393.0897572,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","header":"x-maas-subscription"} {"level":"info","ts":1789741393.0897684,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","header":"x-maas-keyname"} {"level":"info","ts":1789741393.0897741,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","header":"x-maas-username"} {"level":"info","ts":1789741393.0897787,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","header":"x-maas-group"} {"level":"info","ts":1789741393.089792,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789741393.0898192,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.0916822,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.0916991,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.0917163,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","EoS":false} {"level":"info","ts":1789741393.091728,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.0917547,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0","EoS":true} {"level":"info","ts":1789741393.091766,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"efa4da8d45aa421f10cc70dedcb6c36f","span_id":"25737e5d13327e7c","x-request-id":"064680f3-c0e5-4c6e-971d-b444ba33fcc0"} {"level":"info","ts":1789741393.2319574,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741393.232075,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"info","ts":1789741393.2320914,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"info","ts":1789741393.232112,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","EoS":true} {"level":"info","ts":1789741393.2321727,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","header":"x-maas-group"} {"level":"info","ts":1789741393.232187,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","header":"x-maas-subscription"} {"level":"info","ts":1789741393.2321966,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","header":"x-maas-username"} {"level":"info","ts":1789741393.2322059,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","header":"x-maas-keyname"} {"level":"info","ts":1789741393.2322285,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789741393.232272,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"info","ts":1789741393.2329526,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"info","ts":1789741393.232969,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"info","ts":1789741393.2329848,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","EoS":false} {"level":"info","ts":1789741393.2329957,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"info","ts":1789741393.23302,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883","EoS":true} {"level":"info","ts":1789741393.2330306,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a737b966508006e136753185b61a0e95","span_id":"5f2a124ea27eeaa2","x-request-id":"db11c7b9-ab62-4b43-a955-dc90d1ad5883"} {"level":"error","ts":1789741406.2199125,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789741407.1117132,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789741407.1118147,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"info","ts":1789741407.1118264,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"info","ts":1789741407.1118386,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","EoS":true} {"level":"info","ts":1789741407.1118605,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","header":"x-maas-username"} {"level":"info","ts":1789741407.111871,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","header":"x-maas-group"} {"level":"info","ts":1789741407.1118774,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","header":"x-maas-keyname"} {"level":"info","ts":1789741407.111882,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","header":"x-maas-subscription"} {"level":"info","ts":1789741407.1118982,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789741407.1119242,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"info","ts":1789741407.1135383,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"info","ts":1789741407.1135619,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"info","ts":1789741407.1135828,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","EoS":false} {"level":"info","ts":1789741407.1135964,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"info","ts":1789741407.1136186,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a","EoS":true} {"level":"info","ts":1789741407.1136265,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1b499546b0e0954b50d8eec18632b6af","span_id":"e0f20f8aa2bf26b3","x-request-id":"74034a2c-3edf-499e-9520-3c1ebad22c8a"} {"level":"error","ts":1789741421.2210362,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}