{"level":"info","ts":1789821514.2380989,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","header":"x-maas-subscription"} {"level":"info","ts":1789821514.2381096,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","header":"x-maas-group"} {"level":"info","ts":1789821514.2381148,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","header":"x-maas-username"} {"level":"info","ts":1789821514.2381191,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","header":"x-maas-keyname"} {"level":"info","ts":1789821514.238135,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789821514.238159,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee"} {"level":"info","ts":1789821514.2391586,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee"} {"level":"info","ts":1789821514.2391784,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee"} {"level":"info","ts":1789821514.2391958,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","EoS":false} {"level":"info","ts":1789821514.2392106,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee"} {"level":"info","ts":1789821514.2392387,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee","EoS":true} {"level":"info","ts":1789821514.2392535,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"fe0409f979ac59f801b1d5ee40c404dd","span_id":"ca35164f2ce641ff","x-request-id":"dd34452d-498d-4999-9c13-64c51a6708ee"} {"level":"info","ts":1789821514.2820003,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789821514.282114,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.282134,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.2821474,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","EoS":true} {"level":"info","ts":1789821514.2821717,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","header":"x-maas-subscription"} {"level":"info","ts":1789821514.2821822,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","header":"x-maas-keyname"} {"level":"info","ts":1789821514.2821887,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","header":"x-maas-username"} {"level":"info","ts":1789821514.282196,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","header":"x-maas-group"} {"level":"info","ts":1789821514.282214,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789821514.2822425,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.283203,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.283222,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.2832358,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","EoS":false} {"level":"info","ts":1789821514.283248,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.283271,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287","EoS":true} {"level":"info","ts":1789821514.2832804,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9bfee9c8fa919d1e15c5eca5c8337616","span_id":"7da70075d05ad9b1","x-request-id":"28148682-1d28-482a-bbe3-78d90c080287"} {"level":"info","ts":1789821514.3180337,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789821514.3181448,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"info","ts":1789821514.318159,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"info","ts":1789821514.318172,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","EoS":true} {"level":"info","ts":1789821514.318197,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","header":"x-maas-subscription"} {"level":"info","ts":1789821514.3182075,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","header":"x-maas-username"} {"level":"info","ts":1789821514.3182125,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","header":"x-maas-group"} {"level":"info","ts":1789821514.3182168,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","header":"x-maas-keyname"} {"level":"info","ts":1789821514.3182316,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789821514.3182576,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"info","ts":1789821514.3190238,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"info","ts":1789821514.3190408,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"info","ts":1789821514.3190527,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","EoS":false} {"level":"info","ts":1789821514.3190625,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"info","ts":1789821514.319089,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7","EoS":true} {"level":"info","ts":1789821514.3190978,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ae400bdd2d007839a58443c0b37e835b","span_id":"a8b76715c9bdddab","x-request-id":"07ac653f-9492-4a37-8bd4-21bd4fd976e7"} {"level":"error","ts":1789821520.4901886,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789821527.5666237,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789821527.566719,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d4f46e472524d4b4b817fa5837d357ba","span_id":"fea530ab0d71fe75","x-request-id":"575eef44-6fe6-47e1-aeab-1c51a99ebeff"} {"level":"info","ts":1789821527.5667336,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d4f46e472524d4b4b817fa5837d357ba","span_id":"fea530ab0d71fe75","x-request-id":"575eef44-6fe6-47e1-aeab-1c51a99ebeff"} {"level":"info","ts":1789821527.566746,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d4f46e472524d4b4b817fa5837d357ba","span_id":"fea530ab0d71fe75","x-request-id":"575eef44-6fe6-47e1-aeab-1c51a99ebeff","EoS":true} {"level":"info","ts":1789821527.566794,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d4f46e472524d4b4b817fa5837d357ba","span_id":"fea530ab0d71fe75","x-request-id":"575eef44-6fe6-47e1-aeab-1c51a99ebeff"} {"level":"info","ts":1789821527.7024314,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789821527.7025497,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"fa6f746a1d6b2edb910339782e509ee6","span_id":"67964af5f455cf0d","x-request-id":"54085f0f-3650-49aa-a5b3-3f0313c7bea3"} {"level":"info","ts":1789821527.7025628,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"fa6f746a1d6b2edb910339782e509ee6","span_id":"67964af5f455cf0d","x-request-id":"54085f0f-3650-49aa-a5b3-3f0313c7bea3"} {"level":"info","ts":1789821527.7025745,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"fa6f746a1d6b2edb910339782e509ee6","span_id":"67964af5f455cf0d","x-request-id":"54085f0f-3650-49aa-a5b3-3f0313c7bea3","EoS":true} {"level":"info","ts":1789821527.7026167,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"fa6f746a1d6b2edb910339782e509ee6","span_id":"67964af5f455cf0d","x-request-id":"54085f0f-3650-49aa-a5b3-3f0313c7bea3"} {"level":"info","ts":1789821527.8473873,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789821527.8475354,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2864cacf26374df91ca4aca26363eef0","span_id":"5f053c770fd5d127","x-request-id":"bbc1aa9f-6ad5-4a3c-a365-e6b6477927af"} {"level":"info","ts":1789821527.8475509,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2864cacf26374df91ca4aca26363eef0","span_id":"5f053c770fd5d127","x-request-id":"bbc1aa9f-6ad5-4a3c-a365-e6b6477927af"} {"level":"info","ts":1789821527.8475654,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2864cacf26374df91ca4aca26363eef0","span_id":"5f053c770fd5d127","x-request-id":"bbc1aa9f-6ad5-4a3c-a365-e6b6477927af","EoS":true} {"level":"info","ts":1789821527.8476126,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2864cacf26374df91ca4aca26363eef0","span_id":"5f053c770fd5d127","x-request-id":"bbc1aa9f-6ad5-4a3c-a365-e6b6477927af"} {"level":"error","ts":1789821530.490595,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789821545.4921653,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789822642.6408944,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822642.6410186,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.6410334,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.641047,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","EoS":true} {"level":"info","ts":1789822642.6410751,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","header":"x-maas-subscription"} {"level":"info","ts":1789822642.6410847,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","header":"x-maas-group"} {"level":"info","ts":1789822642.6410909,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","header":"x-maas-username"} {"level":"info","ts":1789822642.6410952,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","header":"x-maas-keyname"} {"level":"info","ts":1789822642.6411095,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789822642.6411366,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.6420112,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.6420312,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.642046,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","EoS":false} {"level":"info","ts":1789822642.6420612,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.642089,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036","EoS":true} {"level":"info","ts":1789822642.6420984,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2953c0d438c3993f7f1a8dc1fa75ad4a","span_id":"fc2252f182d70ea9","x-request-id":"b5b3a0f9-5d12-4091-93cf-44761e1a4036"} {"level":"info","ts":1789822642.681663,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822642.6817636,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"info","ts":1789822642.6817772,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"info","ts":1789822642.6817882,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","EoS":true} {"level":"info","ts":1789822642.6818101,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","header":"x-maas-keyname"} {"level":"info","ts":1789822642.6818206,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","header":"x-maas-group"} {"level":"info","ts":1789822642.6818252,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","header":"x-maas-username"} {"level":"info","ts":1789822642.6818295,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","header":"x-maas-subscription"} {"level":"info","ts":1789822642.6818445,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789822642.681869,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"info","ts":1789822642.6826959,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"info","ts":1789822642.6827092,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"info","ts":1789822642.6827197,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","EoS":false} {"level":"info","ts":1789822642.6827295,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"info","ts":1789822642.6827526,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196","EoS":true} {"level":"info","ts":1789822642.6827652,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7e4eba44554f28f0f680873ec78053b6","span_id":"05174a93093b2ebe","x-request-id":"fa58edfb-c602-4215-a811-9156789b2196"} {"level":"error","ts":1789822655.6246252,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789822954.862526,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c886ff41-1159-4c26-8bc9-d74f303d5b16","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789822954.8625672,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8c873ffb-cb21-4d96-a79d-d43acb68faa8"} {"level":"error","ts":1789822954.8625786,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c886ff41-1159-4c26-8bc9-d74f303d5b16","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.8626711,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c886ff41-1159-4c26-8bc9-d74f303d5b16"} {"level":"error","ts":1789822954.867762,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8c873ffb-cb21-4d96-a79d-d43acb68faa8","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.8678231,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"34488411-3c37-4f78-80c9-d471cfa33818","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789822954.8678467,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"34488411-3c37-4f78-80c9-d471cfa33818","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.8678663,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"34488411-3c37-4f78-80c9-d471cfa33818"} {"level":"info","ts":1789822954.872724,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d38ebf8c-7a69-4e06-815d-f1e0c4eef8c0","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789822954.8727467,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d38ebf8c-7a69-4e06-815d-f1e0c4eef8c0","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.8727622,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d38ebf8c-7a69-4e06-815d-f1e0c4eef8c0"} {"level":"error","ts":1789822954.872907,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8c873ffb-cb21-4d96-a79d-d43acb68faa8","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.872971,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1c750ea2-571a-452f-9b4f-d8ef8a1ebc25"} {"level":"error","ts":1789822954.8729985,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1c750ea2-571a-452f-9b4f-d8ef8a1ebc25","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822954.877027,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"1c750ea2-571a-452f-9b4f-d8ef8a1ebc25","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.8872325,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"88d280cf-8e5f-49b0-85d4-7d3813ec8975"} {"level":"error","ts":1789822954.8872654,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"88d280cf-8e5f-49b0-85d4-7d3813ec8975","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822954.8920445,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"88d280cf-8e5f-49b0-85d4-7d3813ec8975","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.9131632,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"83dcf8d6-6b78-45c2-938e-4f7770ed120f"} {"level":"error","ts":1789822954.9131913,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"83dcf8d6-6b78-45c2-938e-4f7770ed120f","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822954.9182992,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"83dcf8d6-6b78-45c2-938e-4f7770ed120f","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822954.9584417,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"cfe8485d-a48d-4489-a97b-0542232ab02d"} {"level":"error","ts":1789822954.9584746,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"cfe8485d-a48d-4489-a97b-0542232ab02d","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822954.9636998,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"cfe8485d-a48d-4489-a97b-0542232ab02d","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822955.0056329,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"020cae00-550d-4a71-8c09-5865b0784e66"} {"level":"error","ts":1789822955.0056667,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"020cae00-550d-4a71-8c09-5865b0784e66","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822955.0056894,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ce814916-4a59-4c40-8d1b-0e81a2126ae0","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789822955.0057156,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ce814916-4a59-4c40-8d1b-0e81a2126ae0","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822955.0057352,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ce814916-4a59-4c40-8d1b-0e81a2126ae0"} {"level":"error","ts":1789822955.0100691,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"020cae00-550d-4a71-8c09-5865b0784e66","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822955.1703002,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ed4a5afa-c73a-4ecb-9da2-73c3f9d91b25"} {"level":"error","ts":1789822955.1703486,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ed4a5afa-c73a-4ecb-9da2-73c3f9d91b25","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822955.177214,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ed4a5afa-c73a-4ecb-9da2-73c3f9d91b25","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822955.4975116,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ee83901f-1c4a-43d4-9afa-5721499d343b"} {"level":"error","ts":1789822955.4975755,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ee83901f-1c4a-43d4-9afa-5721499d343b","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822955.5051305,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ee83901f-1c4a-43d4-9afa-5721499d343b","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822956.1462078,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"668c7343-00cf-4223-b754-255fa609236a"} {"level":"error","ts":1789822956.1462574,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"668c7343-00cf-4223-b754-255fa609236a","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822956.1516345,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"668c7343-00cf-4223-b754-255fa609236a","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822957.4321647,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f3acd63f-2e35-42b7-9be3-710dbfe3498e"} {"level":"error","ts":1789822957.4322178,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f3acd63f-2e35-42b7-9be3-710dbfe3498e","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822957.4371946,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f3acd63f-2e35-42b7-9be3-710dbfe3498e","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822959.9975536,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"06f25715-d377-4d7b-94b1-c5b230b56bc4"} {"level":"error","ts":1789822959.997606,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"06f25715-d377-4d7b-94b1-c5b230b56bc4","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789822960.0040529,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"06f25715-d377-4d7b-94b1-c5b230b56bc4","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822960.0060968,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e84aba5b-a7e6-43d3-ad88-18b2e3795ef2","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789822960.0061367,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e84aba5b-a7e6-43d3-ad88-18b2e3795ef2","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789822960.0061681,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e84aba5b-a7e6-43d3-ad88-18b2e3795ef2"} {"level":"info","ts":1789822960.3026056,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822960.3027349,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.3027513,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.3027663,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","EoS":true} {"level":"info","ts":1789822960.3027914,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","header":"x-maas-group"} {"level":"info","ts":1789822960.3028016,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","header":"x-maas-keyname"} {"level":"info","ts":1789822960.302807,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","header":"x-maas-subscription"} {"level":"info","ts":1789822960.3028138,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","header":"x-maas-username"} {"level":"info","ts":1789822960.3028276,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822960.302852,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.3037596,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.3037794,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.3037941,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","EoS":false} {"level":"info","ts":1789822960.3038063,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.3038313,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6","EoS":true} {"level":"info","ts":1789822960.3038397,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"24fa7c139d1cdb23df3369372d19e7da","span_id":"302bb86f0f4b1e3b","x-request-id":"3827937a-4617-4fcb-93d4-cbd4b04dc9f6"} {"level":"info","ts":1789822960.4087775,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822960.4088795,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4088953,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4089067,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","EoS":true} {"level":"info","ts":1789822960.4089303,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","header":"x-maas-username"} {"level":"info","ts":1789822960.4089417,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","header":"x-maas-subscription"} {"level":"info","ts":1789822960.4089472,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","header":"x-maas-keyname"} {"level":"info","ts":1789822960.4089527,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","header":"x-maas-group"} {"level":"info","ts":1789822960.4089675,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822960.4089925,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4097729,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4097857,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4097974,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","EoS":false} {"level":"info","ts":1789822960.4098072,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4098296,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235","EoS":true} {"level":"info","ts":1789822960.4098463,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c700d5fa6046b3a6a51f2d35af34a5cc","span_id":"227dd2b85d9fd925","x-request-id":"485a8792-c837-4bb2-840a-958f6a65a235"} {"level":"info","ts":1789822960.4984224,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822960.4985728,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.4985871,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.498601,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","EoS":true} {"level":"info","ts":1789822960.498628,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","header":"x-maas-username"} {"level":"info","ts":1789822960.4986377,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","header":"x-maas-group"} {"level":"info","ts":1789822960.4986427,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","header":"x-maas-keyname"} {"level":"info","ts":1789822960.4986467,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","header":"x-maas-subscription"} {"level":"info","ts":1789822960.4986625,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822960.498688,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.4994264,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.4994385,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.4994493,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","EoS":false} {"level":"info","ts":1789822960.499459,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.4994822,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e","EoS":true} {"level":"info","ts":1789822960.4995074,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9a6399998d73726daed4523b6eed5509","span_id":"cfc9c077bf60d919","x-request-id":"bff238ae-197c-4bf7-b8b5-bf3a88f9694e"} {"level":"info","ts":1789822960.6327133,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"37b1e8ef-e8fa-48b4-bf09-4c0fd5c49552","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789822960.6327522,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"37b1e8ef-e8fa-48b4-bf09-4c0fd5c49552","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789822960.632761,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"07f5bb94-6c02-441c-990f-883a07a7df31"} {"level":"error","ts":1789822970.6595504,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789822983.4601355,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822983.46025,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.4602695,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.4602826,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","EoS":true} {"level":"info","ts":1789822983.4603076,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","header":"x-maas-subscription"} {"level":"info","ts":1789822983.4603174,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","header":"x-maas-keyname"} {"level":"info","ts":1789822983.4603226,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","header":"x-maas-username"} {"level":"info","ts":1789822983.4603276,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","header":"x-maas-group"} {"level":"info","ts":1789822983.4603431,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822983.4603698,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.4727585,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.4727843,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.4728036,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","EoS":false} {"level":"info","ts":1789822983.4728165,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.4728382,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357","EoS":true} {"level":"info","ts":1789822983.4728482,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3c0d23b9240851e6b89d371f9741f45e","span_id":"a24e9db59035e17d","x-request-id":"450f7f5d-defd-432c-b91c-6193025f0357"} {"level":"info","ts":1789822983.6127012,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822983.6128027,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.6128163,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.612827,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","EoS":true} {"level":"info","ts":1789822983.612851,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","header":"x-maas-subscription"} {"level":"info","ts":1789822983.6128652,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","header":"x-maas-keyname"} {"level":"info","ts":1789822983.61287,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","header":"x-maas-username"} {"level":"info","ts":1789822983.6128747,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","header":"x-maas-group"} {"level":"info","ts":1789822983.6128888,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822983.6129134,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.6145473,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.6145718,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.614584,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","EoS":false} {"level":"info","ts":1789822983.6145976,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.6146188,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874","EoS":true} {"level":"info","ts":1789822983.6146288,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4a621a4cea8bf8e44bfd7052bc47e88a","span_id":"4b6c22c7adfe342c","x-request-id":"452a47bf-8562-4c70-b676-1df574abd874"} {"level":"info","ts":1789822983.7487752,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822983.7488785,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.748893,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.7489097,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","EoS":true} {"level":"info","ts":1789822983.7489336,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","header":"x-maas-subscription"} {"level":"info","ts":1789822983.7489438,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","header":"x-maas-group"} {"level":"info","ts":1789822983.7489495,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","header":"x-maas-username"} {"level":"info","ts":1789822983.7489543,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","header":"x-maas-keyname"} {"level":"info","ts":1789822983.7489684,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822983.7489986,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.749687,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.749712,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.749733,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","EoS":false} {"level":"info","ts":1789822983.749749,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.7497873,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05","EoS":true} {"level":"info","ts":1789822983.7497988,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6bd5f2243fcffa1263d05bb61728aa7e","span_id":"ba44b9a8174335ac","x-request-id":"4701f0b5-a71c-484f-8282-e1c801239d05"} {"level":"info","ts":1789822983.8872552,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822983.887358,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822983.8873696,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822983.887381,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","EoS":true} {"level":"info","ts":1789822983.8874037,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","header":"x-maas-username"} {"level":"info","ts":1789822983.8874133,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","header":"x-maas-subscription"} {"level":"info","ts":1789822983.887418,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","header":"x-maas-group"} {"level":"info","ts":1789822983.887423,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","header":"x-maas-keyname"} {"level":"info","ts":1789822983.8874385,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822983.887465,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822983.8881135,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822983.8881328,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822983.8881462,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","EoS":false} {"level":"info","ts":1789822983.8881576,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822983.888183,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297","EoS":true} {"level":"info","ts":1789822983.8881924,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3e504e9d2a532872be1e263e2caf7d5c","span_id":"03cb59a0994ebfcf","x-request-id":"d58934a0-e061-4d26-9b00-102e800c4297"} {"level":"info","ts":1789822984.022869,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822984.0229707,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"info","ts":1789822984.0229847,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"info","ts":1789822984.023,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","EoS":true} {"level":"info","ts":1789822984.0230303,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","header":"x-maas-group"} {"level":"info","ts":1789822984.0230403,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","header":"x-maas-subscription"} {"level":"info","ts":1789822984.0230455,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","header":"x-maas-username"} {"level":"info","ts":1789822984.02305,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","header":"x-maas-keyname"} {"level":"info","ts":1789822984.0230641,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822984.0230913,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"info","ts":1789822984.023835,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"info","ts":1789822984.0238612,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"info","ts":1789822984.02388,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","EoS":false} {"level":"info","ts":1789822984.023901,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"info","ts":1789822984.023937,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34","EoS":true} {"level":"info","ts":1789822984.0239527,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"910768efc2f2dbcf0a60cc69fd32cabe","span_id":"cd563b21b1388626","x-request-id":"5dcbaf37-f29d-443c-8914-2bd96873ee34"} {"level":"error","ts":1789822995.6610265,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789822998.1462145,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822998.1463153,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.146329,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.146346,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","EoS":true} {"level":"info","ts":1789822998.1463766,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","header":"x-maas-username"} {"level":"info","ts":1789822998.1463907,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","header":"x-maas-subscription"} {"level":"info","ts":1789822998.146399,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","header":"x-maas-group"} {"level":"info","ts":1789822998.1464055,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","header":"x-maas-keyname"} {"level":"info","ts":1789822998.1464274,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822998.1464624,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.1477625,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.1477816,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.1477995,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","EoS":false} {"level":"info","ts":1789822998.1478124,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.1478338,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c","EoS":true} {"level":"info","ts":1789822998.1478426,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ccaf08e484aec78c425547f399ddd6f1","span_id":"2f4b281831132751","x-request-id":"b421351c-d39d-4d87-ae68-6f6dd1eb911c"} {"level":"info","ts":1789822998.1793053,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822998.17941,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.1794252,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.179438,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","EoS":true} {"level":"info","ts":1789822998.179461,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","header":"x-maas-keyname"} {"level":"info","ts":1789822998.179472,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","header":"x-maas-group"} {"level":"info","ts":1789822998.1794772,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","header":"x-maas-subscription"} {"level":"info","ts":1789822998.1794822,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","header":"x-maas-username"} {"level":"info","ts":1789822998.179516,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822998.179545,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.1802168,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.1802344,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.1802478,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","EoS":false} {"level":"info","ts":1789822998.1802592,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.1802819,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4","EoS":true} {"level":"info","ts":1789822998.1802914,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"33c06cffb14803f4eee211cc4925ea3e","span_id":"8d9c7dcef759609a","x-request-id":"7eadfe64-29ad-4960-82ce-b3fe5372f5c4"} {"level":"info","ts":1789822998.279406,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822998.279528,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e"} {"level":"info","ts":1789822998.2800946,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e"} {"level":"info","ts":1789822998.2801142,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e"} {"level":"info","ts":1789822998.2801268,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e","EoS":false} {"level":"info","ts":1789822998.2801397,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e"} {"level":"info","ts":1789822998.2801607,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e","EoS":true} {"level":"info","ts":1789822998.2801704,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e48c0262f01a748bcc55fe1271a332f7","span_id":"29a25e435ac95317","x-request-id":"61849759-3cfb-4061-a24a-722aec4bb88e"} {"level":"info","ts":1789822999.0336282,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822999.0337274,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.0337424,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.0337574,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","EoS":true} {"level":"info","ts":1789822999.0337832,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","header":"x-maas-subscription"} {"level":"info","ts":1789822999.033793,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","header":"x-maas-group"} {"level":"info","ts":1789822999.0337994,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","header":"x-maas-keyname"} {"level":"info","ts":1789822999.0338051,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","header":"x-maas-username"} {"level":"info","ts":1789822999.0338223,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822999.03385,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.0346682,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.0346878,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.034699,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","EoS":false} {"level":"info","ts":1789822999.0347137,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.034739,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2","EoS":true} {"level":"info","ts":1789822999.034749,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0046900c49e53d6c648e0d75189a4996","span_id":"c789731c8268ae2f","x-request-id":"3da317a2-8629-4bbb-b363-69929b9f9cc2"} {"level":"info","ts":1789822999.9489331,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789822999.9490352,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789822999.949048,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789822999.9490592,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","EoS":true} {"level":"info","ts":1789822999.949083,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","header":"x-maas-subscription"} {"level":"info","ts":1789822999.9490924,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","header":"x-maas-group"} {"level":"info","ts":1789822999.9490974,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","header":"x-maas-keyname"} {"level":"info","ts":1789822999.949102,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","header":"x-maas-username"} {"level":"info","ts":1789822999.9491162,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789822999.9491417,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789822999.9500072,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789822999.9500272,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789822999.9500427,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","EoS":false} {"level":"info","ts":1789822999.9500601,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789822999.9500854,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02","EoS":true} {"level":"info","ts":1789822999.950095,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"42da96487c751352c0afa87b089a9c65","span_id":"3f5f181c32fdc03b","x-request-id":"51fe4e5b-9d20-4ac2-a6df-7b9afe0a4d02"} {"level":"info","ts":1789823000.6680613,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823000.6681604,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823000.6681736,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823000.6681893,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","EoS":true} {"level":"info","ts":1789823000.668216,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","header":"x-maas-username"} {"level":"info","ts":1789823000.668226,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","header":"x-maas-subscription"} {"level":"info","ts":1789823000.6682317,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","header":"x-maas-group"} {"level":"info","ts":1789823000.6682365,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","header":"x-maas-keyname"} {"level":"info","ts":1789823000.6682527,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789823000.668282,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823000.66911,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823000.66913,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823000.6691449,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","EoS":false} {"level":"info","ts":1789823000.6691566,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823000.6691804,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c","EoS":true} {"level":"info","ts":1789823000.6691926,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7a7e31b679b1467cb0841107752c0f7b","span_id":"0328d05db4cd56f7","x-request-id":"dcbc37bd-4f8c-40b2-9bad-7ea9f758713c"} {"level":"info","ts":1789823001.7086751,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823001.7087812,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"info","ts":1789823001.7087939,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"info","ts":1789823001.7088056,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","EoS":true} {"level":"info","ts":1789823001.7088296,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","header":"x-maas-username"} {"level":"info","ts":1789823001.7088394,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","header":"x-maas-subscription"} {"level":"info","ts":1789823001.7088451,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","header":"x-maas-group"} {"level":"info","ts":1789823001.70885,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","header":"x-maas-keyname"} {"level":"info","ts":1789823001.708866,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789823001.7088916,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"info","ts":1789823001.7097082,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"info","ts":1789823001.7097268,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"info","ts":1789823001.7097416,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","EoS":false} {"level":"info","ts":1789823001.709758,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"info","ts":1789823001.7097843,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309","EoS":true} {"level":"info","ts":1789823001.709797,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7e45ffbb8febf41dec5f55b6153a6efe","span_id":"85f9ef8e7f2dc26c","x-request-id":"7d32d80e-0cb1-4db5-8092-b3fc492a9309"} {"level":"error","ts":1789823010.66257,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789823019.164243,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823019.1643507,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"info","ts":1789823019.1643662,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"info","ts":1789823019.1643822,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","EoS":true} {"level":"info","ts":1789823019.164406,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","header":"x-maas-username"} {"level":"info","ts":1789823019.1644166,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","header":"x-maas-group"} {"level":"info","ts":1789823019.164422,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","header":"x-maas-subscription"} {"level":"info","ts":1789823019.1644266,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","header":"x-maas-keyname"} {"level":"info","ts":1789823019.164442,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789823019.164472,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"info","ts":1789823019.1653013,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"info","ts":1789823019.1653275,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"info","ts":1789823019.1653438,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","EoS":false} {"level":"info","ts":1789823019.1653605,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"info","ts":1789823019.1653917,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7","EoS":true} {"level":"info","ts":1789823019.1654024,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3db38b44d35d7111d8991214c33fe66e","span_id":"099e9fcfac25dcc1","x-request-id":"3d05d9d5-d98f-4aa3-aded-2d9c7eefc6d7"} {"level":"error","ts":1789823025.6638582,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789823040.6656308,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789823110.3110187,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823110.3111393,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"info","ts":1789823110.3111537,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"info","ts":1789823110.3111744,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","EoS":true} {"level":"info","ts":1789823110.3112109,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","header":"x-maas-keyname"} {"level":"info","ts":1789823110.311225,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","header":"x-maas-group"} {"level":"info","ts":1789823110.3112333,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","header":"x-maas-username"} {"level":"info","ts":1789823110.3112414,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","header":"x-maas-subscription"} {"level":"info","ts":1789823110.3112633,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789823110.3112986,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"info","ts":1789823110.3240125,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"info","ts":1789823110.3240337,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"info","ts":1789823110.3240492,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","EoS":false} {"level":"info","ts":1789823110.3240612,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"info","ts":1789823110.3240824,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d","EoS":true} {"level":"info","ts":1789823110.3240893,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8eebeb58098cfb807b9d8f8098d75074","span_id":"ae35b1a5563b894e","x-request-id":"e760cf24-76b0-41ae-8884-557ac0ab906d"} {"level":"error","ts":1789823120.6751142,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789823192.8381186,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823192.8382394,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"info","ts":1789823192.8382547,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"info","ts":1789823192.8382678,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","EoS":true} {"level":"info","ts":1789823192.8382921,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","header":"x-maas-keyname"} {"level":"info","ts":1789823192.838303,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","header":"x-maas-username"} {"level":"info","ts":1789823192.8383079,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","header":"x-maas-subscription"} {"level":"info","ts":1789823192.8383126,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","header":"x-maas-group"} {"level":"info","ts":1789823192.8383288,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789823192.8383546,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"info","ts":1789823192.8400126,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"info","ts":1789823192.8400369,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"info","ts":1789823192.8400574,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","EoS":false} {"level":"info","ts":1789823192.8400757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"info","ts":1789823192.8401146,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca","EoS":true} {"level":"info","ts":1789823192.8401258,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dffafad939eeebbde874c1e7f2bf5676","span_id":"abe8fffcd2425938","x-request-id":"1ba8d1cf-f74a-4fc0-84fe-142c089964ca"} {"level":"error","ts":1789823205.684753,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789823206.43902,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823206.439129,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.439143,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.439156,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","EoS":true} {"level":"info","ts":1789823206.4391818,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","header":"x-maas-group"} {"level":"info","ts":1789823206.4391913,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","header":"x-maas-username"} {"level":"info","ts":1789823206.4391966,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","header":"x-maas-keyname"} {"level":"info","ts":1789823206.4392014,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","header":"x-maas-subscription"} {"level":"info","ts":1789823206.4392176,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789823206.4392676,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.4518187,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.4518447,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.4518616,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","EoS":false} {"level":"info","ts":1789823206.4518757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.4519076,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","EoS":false} {"level":"info","ts":1789823206.4519196,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.451932,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5","EoS":true} {"level":"info","ts":1789823206.4519386,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2f88271f14fe8799fc5d28ee188f453e","span_id":"61214569f788af1c","x-request-id":"55701d8a-8f3c-431e-9c4c-b696cbbc80d5"} {"level":"info","ts":1789823206.5874467,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823206.587574,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.587589,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.587601,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","EoS":true} {"level":"info","ts":1789823206.58762,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","header":"x-maas-group"} {"level":"info","ts":1789823206.5876303,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","header":"x-maas-username"} {"level":"info","ts":1789823206.5876355,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","header":"x-maas-keyname"} {"level":"info","ts":1789823206.5876405,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","header":"x-maas-subscription"} {"level":"info","ts":1789823206.5876553,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789823206.587689,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.5894203,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.589441,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.589464,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","EoS":false} {"level":"info","ts":1789823206.5894766,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.5895143,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6","EoS":true} {"level":"info","ts":1789823206.5895293,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f3e1d8d9aa12b0a507cf69d2aacc271b","span_id":"da0f5fc1d9761252","x-request-id":"1ce60352-e9c0-4278-90e2-a8a012537fe6"} {"level":"info","ts":1789823206.7298703,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823206.7299821,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823206.7299955,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823206.730013,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","EoS":true} {"level":"info","ts":1789823206.7300358,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","header":"x-maas-username"} {"level":"info","ts":1789823206.7300458,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","header":"x-maas-group"} {"level":"info","ts":1789823206.7300515,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","header":"x-maas-keyname"} {"level":"info","ts":1789823206.730056,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","header":"x-maas-subscription"} {"level":"info","ts":1789823206.7300718,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789823206.730102,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823206.730793,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823206.7308154,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823206.7308347,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","EoS":false} {"level":"info","ts":1789823206.7308517,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823206.7308838,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012","EoS":true} {"level":"info","ts":1789823206.7308936,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"533f6e3aabfb3af2069579eac4ae2acd","span_id":"6981d042d5433dd1","x-request-id":"e88c8c97-267f-43ce-9b45-edf74aae0012"} {"level":"info","ts":1789823220.531735,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789823220.5318375,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"info","ts":1789823220.5318506,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"info","ts":1789823220.5318625,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","EoS":true} {"level":"info","ts":1789823220.5318828,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","header":"x-maas-subscription"} {"level":"info","ts":1789823220.5318906,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","header":"x-maas-username"} {"level":"info","ts":1789823220.5318959,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","header":"x-maas-keyname"} {"level":"info","ts":1789823220.5319002,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","header":"x-maas-group"} {"level":"info","ts":1789823220.5319147,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789823220.5319417,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"info","ts":1789823220.5333662,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"info","ts":1789823220.533383,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"info","ts":1789823220.5333955,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","EoS":false} {"level":"info","ts":1789823220.5334089,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"info","ts":1789823220.533437,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8","EoS":true} {"level":"info","ts":1789823220.5334501,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"32293dc6bcc748246225ad3e0224d644","span_id":"779def3b5f09db51","x-request-id":"99d86e2e-5d8e-4196-9605-2be42f07c0a8"} {"level":"error","ts":1789823220.6866584,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}