{"level":"info","ts":1789866078.233112,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a84d0fe13bfa9884c00d397d7e17df8f","span_id":"04d95b0e14d6cc62","x-request-id":"3aecfd92-540a-4518-a5b4-8c68ca0ff2a4"}
{"level":"info","ts":1789866078.2331307,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a84d0fe13bfa9884c00d397d7e17df8f","span_id":"04d95b0e14d6cc62","x-request-id":"3aecfd92-540a-4518-a5b4-8c68ca0ff2a4","EoS":false}
{"level":"info","ts":1789866078.2331412,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a84d0fe13bfa9884c00d397d7e17df8f","span_id":"04d95b0e14d6cc62","x-request-id":"3aecfd92-540a-4518-a5b4-8c68ca0ff2a4"}
{"level":"info","ts":1789866078.2331617,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a84d0fe13bfa9884c00d397d7e17df8f","span_id":"04d95b0e14d6cc62","x-request-id":"3aecfd92-540a-4518-a5b4-8c68ca0ff2a4","EoS":true}
{"level":"info","ts":1789866078.2331715,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a84d0fe13bfa9884c00d397d7e17df8f","span_id":"04d95b0e14d6cc62","x-request-id":"3aecfd92-540a-4518-a5b4-8c68ca0ff2a4"}
{"level":"info","ts":1789866078.426406,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789866078.4265208,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"info","ts":1789866078.4265358,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"info","ts":1789866078.4265485,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","EoS":true}
{"level":"info","ts":1789866078.426606,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","header":"x-maas-keyname"}
{"level":"info","ts":1789866078.4266632,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","header":"x-maas-subscription"}
{"level":"info","ts":1789866078.426673,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","header":"x-maas-group"}
{"level":"info","ts":1789866078.4266782,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","header":"x-maas-username"}
{"level":"info","ts":1789866078.4266956,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789866078.426725,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"info","ts":1789866078.4277282,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"info","ts":1789866078.4277885,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"info","ts":1789866078.4278355,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","EoS":false}
{"level":"info","ts":1789866078.4278529,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"info","ts":1789866078.4279137,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4","EoS":true}
{"level":"info","ts":1789866078.427928,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7ccec03b4e02e08aaa948dbe263701a9","span_id":"22e82a95c1df76c3","x-request-id":"80cef328-9f58-46d5-8a8c-26f8427f11c4"}
{"level":"error","ts":1789866081.8055356,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789866091.6085815,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789866091.608664,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3cfd3e2ce9b3250d60a5a09f3760322c","span_id":"5a9b4635e7f2ed7b","x-request-id":"474014cd-86fe-491a-97c9-048508d4d50e"}
{"level":"info","ts":1789866091.6086762,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3cfd3e2ce9b3250d60a5a09f3760322c","span_id":"5a9b4635e7f2ed7b","x-request-id":"474014cd-86fe-491a-97c9-048508d4d50e"}
{"level":"info","ts":1789866091.6086867,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3cfd3e2ce9b3250d60a5a09f3760322c","span_id":"5a9b4635e7f2ed7b","x-request-id":"474014cd-86fe-491a-97c9-048508d4d50e","EoS":true}
{"level":"info","ts":1789866091.60873,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3cfd3e2ce9b3250d60a5a09f3760322c","span_id":"5a9b4635e7f2ed7b","x-request-id":"474014cd-86fe-491a-97c9-048508d4d50e"}
{"level":"info","ts":1789866091.754226,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789866091.7543125,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4b102a68eb9cdc8a356b64b0a115d3c1","span_id":"25fd87782d9324ff","x-request-id":"009be278-0d42-4d6f-8b10-010ca2a2fa8b"}
{"level":"info","ts":1789866091.7543242,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4b102a68eb9cdc8a356b64b0a115d3c1","span_id":"25fd87782d9324ff","x-request-id":"009be278-0d42-4d6f-8b10-010ca2a2fa8b"}
{"level":"info","ts":1789866091.754335,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4b102a68eb9cdc8a356b64b0a115d3c1","span_id":"25fd87782d9324ff","x-request-id":"009be278-0d42-4d6f-8b10-010ca2a2fa8b","EoS":true}
{"level":"info","ts":1789866091.754376,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4b102a68eb9cdc8a356b64b0a115d3c1","span_id":"25fd87782d9324ff","x-request-id":"009be278-0d42-4d6f-8b10-010ca2a2fa8b"}
{"level":"info","ts":1789866091.8965373,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789866091.8966238,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9d6842c47c1c0c36674a282b70c56800","span_id":"82905996624a0345","x-request-id":"6052b78a-2cad-4c41-a6e9-c2db45736d4a"}
{"level":"info","ts":1789866091.8966377,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9d6842c47c1c0c36674a282b70c56800","span_id":"82905996624a0345","x-request-id":"6052b78a-2cad-4c41-a6e9-c2db45736d4a"}
{"level":"info","ts":1789866091.8966486,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9d6842c47c1c0c36674a282b70c56800","span_id":"82905996624a0345","x-request-id":"6052b78a-2cad-4c41-a6e9-c2db45736d4a","EoS":true}
{"level":"info","ts":1789866091.896687,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9d6842c47c1c0c36674a282b70c56800","span_id":"82905996624a0345","x-request-id":"6052b78a-2cad-4c41-a6e9-c2db45736d4a"}
{"level":"error","ts":1789866096.8060076,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"error","ts":1789866111.8073,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867140.5082827,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867140.508409,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.508423,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.5084352,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","EoS":true}
{"level":"info","ts":1789867140.5084624,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","header":"x-maas-username"}
{"level":"info","ts":1789867140.5084739,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","header":"x-maas-subscription"}
{"level":"info","ts":1789867140.5084786,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","header":"x-maas-group"}
{"level":"info","ts":1789867140.5084834,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","header":"x-maas-keyname"}
{"level":"info","ts":1789867140.508498,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"}
{"level":"info","ts":1789867140.5085256,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.5096128,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.509639,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.509661,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","EoS":false}
{"level":"info","ts":1789867140.509676,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.5097008,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3","EoS":true}
{"level":"info","ts":1789867140.5097098,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c497f6f1cae74429e9229a63ee87c040","span_id":"b3e39a1d404d30f5","x-request-id":"f70335aa-3c20-4ea5-b5b1-cc57e50940f3"}
{"level":"info","ts":1789867140.5497305,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867140.549826,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"info","ts":1789867140.5498388,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"info","ts":1789867140.5498493,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","EoS":true}
{"level":"info","ts":1789867140.5498893,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","header":"x-maas-group"}
{"level":"info","ts":1789867140.5499,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","header":"x-maas-username"}
{"level":"info","ts":1789867140.5499063,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","header":"x-maas-subscription"}
{"level":"info","ts":1789867140.5499113,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","header":"x-maas-keyname"}
{"level":"info","ts":1789867140.549925,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"}
{"level":"info","ts":1789867140.5499527,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"info","ts":1789867140.5509727,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"info","ts":1789867140.5509856,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"info","ts":1789867140.5509968,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","EoS":false}
{"level":"info","ts":1789867140.551014,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"info","ts":1789867140.5510368,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f","EoS":true}
{"level":"info","ts":1789867140.551051,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d6a2ae500cc316c1550bb858fc6f09e0","span_id":"6389a1bb72f72cc2","x-request-id":"5cf90071-4dd4-4cce-a5d8-a632c355690f"}
{"level":"error","ts":1789867151.9368968,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867441.4646618,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a30916fd-a542-484c-ad78-aca2a726b690","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789867441.4647214,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a30916fd-a542-484c-ad78-aca2a726b690","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.4647424,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a30916fd-a542-484c-ad78-aca2a726b690"}
{"level":"info","ts":1789867441.4646618,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"49bb42eb-ba33-466d-b0bc-a286602e73bd"}
{"level":"info","ts":1789867441.4717994,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"918c8371-31a8-4e2f-8686-a09cf1899d2b","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789867441.471826,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"918c8371-31a8-4e2f-8686-a09cf1899d2b","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.4718442,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"918c8371-31a8-4e2f-8686-a09cf1899d2b"}
{"level":"error","ts":1789867441.4729164,"caller":"externalmodel/reconciler.go:211","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"49bb42eb-ba33-466d-b0bc-a286602e73bd","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-x-api-key-trigger\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:211\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.4729505,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"49bb42eb-ba33-466d-b0bc-a286602e73bd","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.478181,"caller":"externalmodel/reconciler.go:211","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"49bb42eb-ba33-466d-b0bc-a286602e73bd","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-x-api-key-trigger\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:211\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.4782152,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"49bb42eb-ba33-466d-b0bc-a286602e73bd","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.4782586,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"227f3805-44f3-4008-bda6-b29214183f52"}
{"level":"error","ts":1789867441.4782784,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"227f3805-44f3-4008-bda6-b29214183f52","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.4791524,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f3e3deaa-1bcc-40fe-a5ca-9d6c6a7de6b3","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789867441.4791784,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f3e3deaa-1bcc-40fe-a5ca-9d6c6a7de6b3","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.4791937,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"f3e3deaa-1bcc-40fe-a5ca-9d6c6a7de6b3"}
{"level":"error","ts":1789867441.4815443,"caller":"externalmodel/reconciler.go:211","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"227f3805-44f3-4008-bda6-b29214183f52","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-x-api-key-trigger\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:211\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.4815805,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"227f3805-44f3-4008-bda6-b29214183f52","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.48162,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d7b9965f-e481-49be-87bc-b7e824fdb8f8"}
{"level":"error","ts":1789867441.4816399,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d7b9965f-e481-49be-87bc-b7e824fdb8f8","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.4852707,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d7b9965f-e481-49be-87bc-b7e824fdb8f8","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.5054948,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c7a8eb93-27ce-46b7-8e9d-48d381298c3b"}
{"level":"error","ts":1789867441.505527,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c7a8eb93-27ce-46b7-8e9d-48d381298c3b","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.5095236,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c7a8eb93-27ce-46b7-8e9d-48d381298c3b","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.5506444,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"23bdc0df-f07b-4356-a7cd-00480b8723a8"}
{"level":"error","ts":1789867441.550676,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"23bdc0df-f07b-4356-a7cd-00480b8723a8","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.5567358,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"23bdc0df-f07b-4356-a7cd-00480b8723a8","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.6063995,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a6a4c94c-9aac-44c4-9d16-912546fe2325","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789867441.6064336,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a6a4c94c-9aac-44c4-9d16-912546fe2325","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.6064498,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"a6a4c94c-9aac-44c4-9d16-912546fe2325"}
{"level":"info","ts":1789867441.6064029,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9cd3eb75-9759-4df2-84d3-649e5783e4f3"}
{"level":"error","ts":1789867441.606484,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9cd3eb75-9759-4df2-84d3-649e5783e4f3","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.6109836,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9cd3eb75-9759-4df2-84d3-649e5783e4f3","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867441.7711606,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4273f63e-3ce9-439c-a160-3a14147bdf91"}
{"level":"error","ts":1789867441.7712245,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4273f63e-3ce9-439c-a160-3a14147bdf91","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867441.7763777,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4273f63e-3ce9-439c-a160-3a14147bdf91","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867442.0969918,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d2c8101f-2007-492a-8f6d-0aa690a08e56"}
{"level":"error","ts":1789867442.0970526,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d2c8101f-2007-492a-8f6d-0aa690a08e56","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867442.1024523,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d2c8101f-2007-492a-8f6d-0aa690a08e56","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867442.7430344,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8db2e8c1-dc2d-4e57-a5f1-f3095f3bc5d5"}
{"level":"error","ts":1789867442.7430933,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8db2e8c1-dc2d-4e57-a5f1-f3095f3bc5d5","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867442.7478905,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"8db2e8c1-dc2d-4e57-a5f1-f3095f3bc5d5","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867444.028711,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"48eae1cf-9c2d-4bcb-90ea-5cbc88906944"}
{"level":"error","ts":1789867444.0287662,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"48eae1cf-9c2d-4bcb-90ea-5cbc88906944","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867444.0342617,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"48eae1cf-9c2d-4bcb-90ea-5cbc88906944","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867446.5947978,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"83690051-daf8-48fb-b1f3-5b73e8f4832e"}
{"level":"error","ts":1789867446.5948806,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"83690051-daf8-48fb-b1f3-5b73e8f4832e","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"error","ts":1789867446.5992734,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"83690051-daf8-48fb-b1f3-5b73e8f4832e","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867446.607366,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6165c25b-a77e-47c6-a99d-f0ded65dfdeb","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"error","ts":1789867446.6074026,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6165c25b-a77e-47c6-a99d-f0ded65dfdeb","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"}
{"level":"info","ts":1789867446.6074324,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6165c25b-a77e-47c6-a99d-f0ded65dfdeb"}
{"level":"info","ts":1789867446.8929944,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867446.893089,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.8931012,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.8931136,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","EoS":true}
{"level":"info","ts":1789867446.8931346,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","header":"x-maas-group"}
{"level":"info","ts":1789867446.8931456,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","header":"x-maas-username"}
{"level":"info","ts":1789867446.8931515,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","header":"x-maas-keyname"}
{"level":"info","ts":1789867446.8931558,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","header":"x-maas-subscription"}
{"level":"info","ts":1789867446.89317,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867446.893195,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.8942266,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.8942451,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.8942654,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","EoS":false}
{"level":"info","ts":1789867446.894282,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.894322,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb","EoS":true}
{"level":"info","ts":1789867446.8943346,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"aebaf12b9215335eb2040e0eb69e3c9d","span_id":"e1ee23ab79f79c2a","x-request-id":"155aa2ad-88af-409e-b99f-a3b9e62f8dcb"}
{"level":"info","ts":1789867446.9986415,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867446.9987185,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867446.9987314,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867446.9987426,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","EoS":true}
{"level":"info","ts":1789867446.9987628,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","header":"x-maas-group"}
{"level":"info","ts":1789867446.9987702,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","header":"x-maas-username"}
{"level":"info","ts":1789867446.998775,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","header":"x-maas-subscription"}
{"level":"info","ts":1789867446.9987803,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","header":"x-maas-keyname"}
{"level":"info","ts":1789867446.9987938,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867446.9988186,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867446.9998145,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867446.9998338,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867446.9998496,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","EoS":false}
{"level":"info","ts":1789867446.999893,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867446.9999301,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9","EoS":true}
{"level":"info","ts":1789867446.9999433,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3b3bd18f731a4b70f4c712261690739b","span_id":"72a0c4ef75015db0","x-request-id":"110e3cd7-9ddf-478f-b827-8430b14519a9"}
{"level":"info","ts":1789867447.0963218,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867447.096412,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.096425,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.0964355,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","EoS":true}
{"level":"info","ts":1789867447.0964532,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","header":"x-maas-group"}
{"level":"info","ts":1789867447.0964603,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","header":"x-maas-subscription"}
{"level":"info","ts":1789867447.0964673,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","header":"x-maas-username"}
{"level":"info","ts":1789867447.0964744,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","header":"x-maas-keyname"}
{"level":"info","ts":1789867447.0964923,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867447.0965202,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.0975087,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.0975232,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.0975351,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","EoS":false}
{"level":"info","ts":1789867447.097555,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.0975778,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff","EoS":true}
{"level":"info","ts":1789867447.0975873,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d766d8c27dd2f9e85e1aeb18731af3a7","span_id":"cd3290612cbe108d","x-request-id":"cec819eb-657e-465f-abe0-c1102014a1ff"}
{"level":"info","ts":1789867447.231547,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"847cb0ea-0e30-4b39-b6e4-a10845884ce1","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"info","ts":1789867447.231582,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"847cb0ea-0e30-4b39-b6e4-a10845884ce1","name":"e2e-x-api-key-trigger","namespace":"llm"}
{"level":"info","ts":1789867447.231587,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"688b31f6-c989-49d1-80c1-1c8603194720"}
{"level":"error","ts":1789867456.9686675,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867469.8884315,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867469.8885322,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867469.8885496,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867469.8885627,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","EoS":true}
{"level":"info","ts":1789867469.8885882,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","header":"x-maas-group"}
{"level":"info","ts":1789867469.8885982,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","header":"x-maas-keyname"}
{"level":"info","ts":1789867469.8886037,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","header":"x-maas-username"}
{"level":"info","ts":1789867469.888611,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","header":"x-maas-subscription"}
{"level":"info","ts":1789867469.8886292,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867469.888657,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867469.902479,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867469.9025111,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867469.902528,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","EoS":false}
{"level":"info","ts":1789867469.9025433,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867469.9025736,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f","EoS":true}
{"level":"info","ts":1789867469.9025815,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ceb46879bfdfc94cd03e599ce09b2e52","span_id":"2cd84bef5f32b4bb","x-request-id":"e93ddb1e-0efc-4ea4-81ba-04ebd02ae11f"}
{"level":"info","ts":1789867470.0404987,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867470.0405958,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.0406096,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.0406268,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","EoS":true}
{"level":"info","ts":1789867470.0406578,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","header":"x-maas-username"}
{"level":"info","ts":1789867470.040673,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","header":"x-maas-keyname"}
{"level":"info","ts":1789867470.0406823,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","header":"x-maas-group"}
{"level":"info","ts":1789867470.0406914,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","header":"x-maas-subscription"}
{"level":"info","ts":1789867470.040712,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867470.0407474,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.0429957,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.043018,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.0430324,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","EoS":false}
{"level":"info","ts":1789867470.043044,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.0430694,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275","EoS":true}
{"level":"info","ts":1789867470.0430787,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a1da0338d54f674a12567265b9cc584a","span_id":"7609e59dea67bb38","x-request-id":"771111f1-1afc-45dd-89ce-f5c5d220b275"}
{"level":"info","ts":1789867470.1860385,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867470.186135,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.1861491,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.186167,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","EoS":true}
{"level":"info","ts":1789867470.1861937,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","header":"x-maas-username"}
{"level":"info","ts":1789867470.1862075,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","header":"x-maas-subscription"}
{"level":"info","ts":1789867470.186217,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","header":"x-maas-group"}
{"level":"info","ts":1789867470.1862264,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","header":"x-maas-keyname"}
{"level":"info","ts":1789867470.1862457,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867470.186279,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.1872423,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.1872597,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.187273,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","EoS":false}
{"level":"info","ts":1789867470.1872835,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.1873064,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e","EoS":true}
{"level":"info","ts":1789867470.1873186,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"caaa54f32c9278d86b94036e59676ae9","span_id":"bad6b8d90bc96c5e","x-request-id":"fac96d54-7a85-4141-b16c-cf8570fd130e"}
{"level":"info","ts":1789867470.3221087,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867470.322211,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.3222246,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.3222423,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","EoS":true}
{"level":"info","ts":1789867470.3222713,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","header":"x-maas-username"}
{"level":"info","ts":1789867470.3222845,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","header":"x-maas-keyname"}
{"level":"info","ts":1789867470.322294,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","header":"x-maas-subscription"}
{"level":"info","ts":1789867470.3223028,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","header":"x-maas-group"}
{"level":"info","ts":1789867470.3223221,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867470.3223493,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.3233304,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.3233483,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.3233588,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","EoS":false}
{"level":"info","ts":1789867470.3233712,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.3233962,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714","EoS":true}
{"level":"info","ts":1789867470.3234046,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"126b472d3ba21eb6c4d95ae4799a7d47","span_id":"3b130efc02f8969b","x-request-id":"064dc895-ac74-4bdb-a28c-d0b35b6de714"}
{"level":"info","ts":1789867470.4629972,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867470.4630742,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"info","ts":1789867470.463087,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"info","ts":1789867470.4630978,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","EoS":true}
{"level":"info","ts":1789867470.4631166,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","header":"x-maas-group"}
{"level":"info","ts":1789867470.4631271,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","header":"x-maas-username"}
{"level":"info","ts":1789867470.463132,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","header":"x-maas-subscription"}
{"level":"info","ts":1789867470.4631362,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","header":"x-maas-keyname"}
{"level":"info","ts":1789867470.4631488,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867470.463172,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"info","ts":1789867470.4640698,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"info","ts":1789867470.4640877,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"info","ts":1789867470.4641058,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","EoS":false}
{"level":"info","ts":1789867470.46412,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"info","ts":1789867470.464154,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da","EoS":true}
{"level":"info","ts":1789867470.4641666,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"770446eaa4a1a8877a40dc52c3c341ab","span_id":"81cdb1b6f9058d81","x-request-id":"2ff60b91-251d-4447-8a17-3d196fd3f0da"}
{"level":"error","ts":1789867471.9701772,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"error","ts":1789867481.9712722,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867485.5319486,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867485.5320675,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.5320868,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.5321007,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","EoS":true}
{"level":"info","ts":1789867485.5321214,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","header":"x-maas-subscription"}
{"level":"info","ts":1789867485.5321295,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","header":"x-maas-keyname"}
{"level":"info","ts":1789867485.532134,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","header":"x-maas-group"}
{"level":"info","ts":1789867485.5321388,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","header":"x-maas-username"}
{"level":"info","ts":1789867485.5321524,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867485.5321746,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.5334218,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.533447,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.5334668,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","EoS":false}
{"level":"info","ts":1789867485.5334804,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.5335085,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b","EoS":true}
{"level":"info","ts":1789867485.5335178,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7d5aac175a2179683dd3e5c5ebc3afe6","span_id":"068fb5a34bd7a4b2","x-request-id":"5dbc9a8c-c67e-42d3-a387-d465a3d4fc8b"}
{"level":"info","ts":1789867485.5671434,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867485.5672088,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.5672197,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.5672383,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","EoS":true}
{"level":"info","ts":1789867485.5672588,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","header":"x-maas-group"}
{"level":"info","ts":1789867485.5672684,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","header":"x-maas-username"}
{"level":"info","ts":1789867485.5672736,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","header":"x-maas-subscription"}
{"level":"info","ts":1789867485.5672781,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","header":"x-maas-keyname"}
{"level":"info","ts":1789867485.5672894,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867485.5673094,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.568278,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.5682902,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.5683012,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","EoS":false}
{"level":"info","ts":1789867485.56831,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.5683286,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2","EoS":true}
{"level":"info","ts":1789867485.568337,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c152bc4786f6afc859a3a50cb8a78b4c","span_id":"03ed2b48f40f5369","x-request-id":"43b4d22c-bb43-449d-9f16-93ba64098ea2"}
{"level":"info","ts":1789867485.6727924,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867485.672902,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50"}
{"level":"info","ts":1789867485.6738176,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50"}
{"level":"info","ts":1789867485.6738322,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50"}
{"level":"info","ts":1789867485.673855,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50","EoS":false}
{"level":"info","ts":1789867485.673895,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50"}
{"level":"info","ts":1789867485.673919,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50","EoS":true}
{"level":"info","ts":1789867485.6739347,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9f983e7cb147b7b0766df82424920da3","span_id":"affbaf2914f9c85f","x-request-id":"026bafab-ac74-43d6-a887-534ca2336e50"}
{"level":"info","ts":1789867486.4276133,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867486.4276984,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867486.4277103,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867486.427723,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","EoS":true}
{"level":"info","ts":1789867486.4277458,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","header":"x-maas-username"}
{"level":"info","ts":1789867486.4277563,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","header":"x-maas-keyname"}
{"level":"info","ts":1789867486.4277613,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","header":"x-maas-group"}
{"level":"info","ts":1789867486.4277658,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","header":"x-maas-subscription"}
{"level":"info","ts":1789867486.427781,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867486.427805,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867486.428807,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867486.4288192,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867486.4288332,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","EoS":false}
{"level":"info","ts":1789867486.4288445,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867486.4288816,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf","EoS":true}
{"level":"info","ts":1789867486.428891,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"d1e2e5795e515cfecbd3e74774970fe5","span_id":"e18b160cc31438bf","x-request-id":"ed734f50-e3c6-46fc-ba49-14c27eee2ecf"}
{"level":"info","ts":1789867487.34565,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867487.345731,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867487.3457432,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867487.3457541,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","EoS":true}
{"level":"info","ts":1789867487.3457725,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","header":"x-maas-group"}
{"level":"info","ts":1789867487.3457828,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","header":"x-maas-subscription"}
{"level":"info","ts":1789867487.345789,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","header":"x-maas-username"}
{"level":"info","ts":1789867487.3457935,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","header":"x-maas-keyname"}
{"level":"info","ts":1789867487.3458066,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867487.3458276,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867487.346855,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867487.3468883,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867487.3469012,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","EoS":false}
{"level":"info","ts":1789867487.3469136,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867487.3469338,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9","EoS":true}
{"level":"info","ts":1789867487.3469427,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2621580b8ba8d29866a7a4319da45d03","span_id":"a2f8e72ce1d880a6","x-request-id":"701213cd-e546-48ef-87f4-30038a4fa1d9"}
{"level":"info","ts":1789867488.0586839,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867488.0587695,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867488.0587826,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867488.0587933,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","EoS":true}
{"level":"info","ts":1789867488.0588236,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","header":"x-maas-keyname"}
{"level":"info","ts":1789867488.0588362,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","header":"x-maas-subscription"}
{"level":"info","ts":1789867488.0588408,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","header":"x-maas-group"}
{"level":"info","ts":1789867488.0588455,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","header":"x-maas-username"}
{"level":"info","ts":1789867488.0588846,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867488.058907,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867488.0598881,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867488.059906,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867488.0599244,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","EoS":false}
{"level":"info","ts":1789867488.0599356,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867488.059958,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6","EoS":true}
{"level":"info","ts":1789867488.0599658,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ac048bada00d7b13763173aecc0154c5","span_id":"15a65f4b6a2bb8f9","x-request-id":"ee7437ef-8386-4d44-942e-359ac73f2ba6"}
{"level":"info","ts":1789867489.0694752,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867489.0695658,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"info","ts":1789867489.0695796,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"info","ts":1789867489.0695913,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","EoS":true}
{"level":"info","ts":1789867489.069612,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","header":"x-maas-username"}
{"level":"info","ts":1789867489.0696216,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","header":"x-maas-keyname"}
{"level":"info","ts":1789867489.069627,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","header":"x-maas-group"}
{"level":"info","ts":1789867489.0696318,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","header":"x-maas-subscription"}
{"level":"info","ts":1789867489.0696461,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867489.0696723,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"info","ts":1789867489.0705824,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"info","ts":1789867489.070602,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"info","ts":1789867489.0706165,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","EoS":false}
{"level":"info","ts":1789867489.0706258,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"info","ts":1789867489.070647,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1","EoS":true}
{"level":"info","ts":1789867489.0706568,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"6e532f43cfd226bc85d1d473966a5623","span_id":"f11da8581303b68d","x-request-id":"6a83bc53-d0c0-48d8-9037-23378da9dda1"}
{"level":"error","ts":1789867496.971843,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867508.479319,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867508.4794075,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"info","ts":1789867508.4794216,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"info","ts":1789867508.4794333,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","EoS":true}
{"level":"info","ts":1789867508.4794521,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","header":"x-maas-username"}
{"level":"info","ts":1789867508.479463,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","header":"x-maas-subscription"}
{"level":"info","ts":1789867508.4794734,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","header":"x-maas-keyname"}
{"level":"info","ts":1789867508.4794807,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","header":"x-maas-group"}
{"level":"info","ts":1789867508.4794948,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","path":"/llm/facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867508.4795184,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"info","ts":1789867508.4804728,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"info","ts":1789867508.4804857,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"info","ts":1789867508.4804966,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","EoS":false}
{"level":"info","ts":1789867508.4805064,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"info","ts":1789867508.4805322,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc","EoS":true}
{"level":"info","ts":1789867508.4805455,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"8f0a5f0e9ced4dc4e24a4cc24fa4f29b","span_id":"8a5f6ba4ac05a620","x-request-id":"fd6dd3f2-54de-4ce6-8876-4a3a55a840fc"}
{"level":"error","ts":1789867511.9721224,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"error","ts":1789867521.9727287,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867599.7898004,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867599.789912,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"info","ts":1789867599.789925,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"info","ts":1789867599.7899373,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","EoS":true}
{"level":"info","ts":1789867599.789958,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","header":"x-maas-username"}
{"level":"info","ts":1789867599.7899678,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","header":"x-maas-group"}
{"level":"info","ts":1789867599.7899733,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","header":"x-maas-subscription"}
{"level":"info","ts":1789867599.7899783,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","header":"x-maas-keyname"}
{"level":"info","ts":1789867599.7899919,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","path":"/llm/e2e-trlp-test-simulated/v1/completions"}
{"level":"info","ts":1789867599.7900183,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"info","ts":1789867599.8034365,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"info","ts":1789867599.803458,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"info","ts":1789867599.8034716,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","EoS":false}
{"level":"info","ts":1789867599.8034823,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"info","ts":1789867599.8035028,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc","EoS":true}
{"level":"info","ts":1789867599.803512,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0378475281382cd23e174f2f29075197","span_id":"679c7875a16a15ea","x-request-id":"e115603c-8cd7-4e0f-a2f5-fe790e6e32dc"}
{"level":"error","ts":1789867611.9828093,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867669.2587636,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867669.258855,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.2589018,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.2589157,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","EoS":true}
{"level":"info","ts":1789867669.2589374,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","header":"x-maas-username"}
{"level":"info","ts":1789867669.2589467,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","header":"x-maas-group"}
{"level":"info","ts":1789867669.2589543,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","header":"x-maas-subscription"}
{"level":"info","ts":1789867669.258961,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","header":"x-maas-keyname"}
{"level":"info","ts":1789867669.2589786,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867669.259015,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.260935,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.260957,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.260971,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","EoS":false}
{"level":"info","ts":1789867669.2609825,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.261006,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b","EoS":true}
{"level":"info","ts":1789867669.2610135,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"67422654820fe181e84f0e32b27e3dc2","span_id":"25057f116b4acaa8","x-request-id":"3cad835d-0f15-44c9-b81e-edc28b01cb1b"}
{"level":"info","ts":1789867669.298688,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867669.2987914,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"info","ts":1789867669.2988062,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"info","ts":1789867669.2988245,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","EoS":true}
{"level":"info","ts":1789867669.2988782,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","header":"x-maas-username"}
{"level":"info","ts":1789867669.2988915,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","header":"x-maas-keyname"}
{"level":"info","ts":1789867669.2989008,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","header":"x-maas-subscription"}
{"level":"info","ts":1789867669.298908,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","header":"x-maas-group"}
{"level":"info","ts":1789867669.2989285,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"}
{"level":"info","ts":1789867669.29896,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"info","ts":1789867669.29995,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"info","ts":1789867669.2999673,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"info","ts":1789867669.29998,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","EoS":false}
{"level":"info","ts":1789867669.2999904,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"info","ts":1789867669.3000147,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889","EoS":true}
{"level":"info","ts":1789867669.3000243,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5f7ee68ebb7752c6a48ff32044253be9","span_id":"63d41390b2077957","x-request-id":"904c04bc-9557-4cb0-9af7-f28ed3970889"}
{"level":"error","ts":1789867681.9916701,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867682.9355986,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867682.935705,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.935718,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.93573,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","EoS":true}
{"level":"info","ts":1789867682.9357507,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","header":"x-maas-username"}
{"level":"info","ts":1789867682.9357617,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","header":"x-maas-group"}
{"level":"info","ts":1789867682.9357667,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","header":"x-maas-keyname"}
{"level":"info","ts":1789867682.9357717,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","header":"x-maas-subscription"}
{"level":"info","ts":1789867682.9357858,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","path":"/llm/e2e-embedding-simulated/v1/embeddings"}
{"level":"info","ts":1789867682.9358115,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.9482768,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.948302,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.9483192,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","EoS":false}
{"level":"info","ts":1789867682.948331,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.94836,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","EoS":false}
{"level":"info","ts":1789867682.9483762,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867682.9483874,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9","EoS":true}
{"level":"info","ts":1789867682.9483943,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"16f47e462585db390e2684b3366b54bb","span_id":"58a5a60c97359109","x-request-id":"afc0a707-75b7-48fd-a760-88a9ff0f24f9"}
{"level":"info","ts":1789867683.0900187,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867683.0901144,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.090126,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.0901384,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","EoS":true}
{"level":"info","ts":1789867683.0901582,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","header":"x-maas-group"}
{"level":"info","ts":1789867683.0901697,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","header":"x-maas-subscription"}
{"level":"info","ts":1789867683.0901742,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","header":"x-maas-keyname"}
{"level":"info","ts":1789867683.0901785,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","header":"x-maas-username"}
{"level":"info","ts":1789867683.0901923,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","path":"/llm/e2e-embedding-simulated/v1/embeddings"}
{"level":"info","ts":1789867683.0902164,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.091173,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.0911908,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.0912082,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","EoS":false}
{"level":"info","ts":1789867683.091221,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.0912468,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f","EoS":true}
{"level":"info","ts":1789867683.0912595,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"49fe8225e3fb06d0ec8965a0a9c5d7dc","span_id":"d0207c822b1b3f9c","x-request-id":"54434d97-4d4a-408a-bf38-70a66ed9744f"}
{"level":"info","ts":1789867683.2284336,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867683.2285216,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"info","ts":1789867683.2285392,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"info","ts":1789867683.2285502,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","EoS":true}
{"level":"info","ts":1789867683.2285676,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","header":"x-maas-keyname"}
{"level":"info","ts":1789867683.2285774,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","header":"x-maas-subscription"}
{"level":"info","ts":1789867683.2285824,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","header":"x-maas-group"}
{"level":"info","ts":1789867683.228587,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","header":"x-maas-username"}
{"level":"info","ts":1789867683.228602,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","path":"/llm/e2e-embedding-simulated/v1/embeddings"}
{"level":"info","ts":1789867683.2286286,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"info","ts":1789867683.2295728,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"info","ts":1789867683.2295897,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"info","ts":1789867683.2296054,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","EoS":false}
{"level":"info","ts":1789867683.229618,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"info","ts":1789867683.229642,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c","EoS":true}
{"level":"info","ts":1789867683.2296512,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a0b332001d97f4b8e1f293a7cb10cee7","span_id":"8ed21308a0124005","x-request-id":"cc41ffbd-312e-4c8e-b0d9-c739daf6672c"}
{"level":"error","ts":1789867696.992186,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}
{"level":"info","ts":1789867699.1378834,"caller":"handlers/server.go:147","msg":"Processing"}
{"level":"info","ts":1789867699.1379902,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.1380026,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.138015,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","EoS":true}
{"level":"info","ts":1789867699.1380353,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","header":"x-maas-keyname"}
{"level":"info","ts":1789867699.1380446,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","header":"x-maas-subscription"}
{"level":"info","ts":1789867699.138051,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","header":"x-maas-username"}
{"level":"info","ts":1789867699.1380575,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","header":"x-maas-group"}
{"level":"info","ts":1789867699.138076,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","path":"/llm/e2e-embedding-simulated/v1/embeddings"}
{"level":"info","ts":1789867699.1381025,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.1401343,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.1401572,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.140171,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","EoS":false}
{"level":"info","ts":1789867699.140186,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.1402135,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","EoS":false}
{"level":"info","ts":1789867699.1402256,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"info","ts":1789867699.1402361,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484","EoS":true}
{"level":"info","ts":1789867699.1402433,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"1ef9b756d19f93fe59a450791a9f8d89","span_id":"e0e4d87e3f8ee21d","x-request-id":"7a47f7b4-77e3-4792-817d-a1502dbb3484"}
{"level":"error","ts":1789867711.992996,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}