{"level":"info","ts":1789792478.8231196,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"080fd59c3f428f53b02633c74fc5a601","span_id":"66a6d466604cf23f","x-request-id":"6a91f3cd-36b9-9938-b1a7-e25830f84a96"} {"level":"info","ts":1789792478.8231575,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"080fd59c3f428f53b02633c74fc5a601","span_id":"66a6d466604cf23f","x-request-id":"6a91f3cd-36b9-9938-b1a7-e25830f84a96"} {"level":"info","ts":1789792478.8231797,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"080fd59c3f428f53b02633c74fc5a601","span_id":"66a6d466604cf23f","x-request-id":"6a91f3cd-36b9-9938-b1a7-e25830f84a96","EoS":false} {"level":"info","ts":1789792478.823197,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"080fd59c3f428f53b02633c74fc5a601","span_id":"66a6d466604cf23f","x-request-id":"6a91f3cd-36b9-9938-b1a7-e25830f84a96"} {"level":"info","ts":1789792478.8232355,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"080fd59c3f428f53b02633c74fc5a601","span_id":"66a6d466604cf23f","x-request-id":"6a91f3cd-36b9-9938-b1a7-e25830f84a96","EoS":true} {"level":"info","ts":1789792478.8232496,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"080fd59c3f428f53b02633c74fc5a601","span_id":"66a6d466604cf23f","x-request-id":"6a91f3cd-36b9-9938-b1a7-e25830f84a96"} {"level":"info","ts":1789792478.8633037,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789792478.863391,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.863404,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.8634164,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","EoS":true} {"level":"info","ts":1789792478.8634381,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","header":"x-maas-group"} {"level":"info","ts":1789792478.8634481,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","header":"x-maas-username"} {"level":"info","ts":1789792478.8634531,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","header":"x-maas-subscription"} {"level":"info","ts":1789792478.8634572,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","header":"x-maas-keyname"} {"level":"info","ts":1789792478.863474,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789792478.8634994,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.8642902,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.8643112,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.8643255,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","EoS":false} {"level":"info","ts":1789792478.864337,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.8643632,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7","EoS":true} {"level":"info","ts":1789792478.8643801,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9b8f5c339506324e2a3c56deb794b819","span_id":"5e0d19845bcc52dd","x-request-id":"e4cf3152-d913-40e6-9ef9-0408e16785f7"} {"level":"info","ts":1789792478.9085283,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789792478.9086423,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.9086583,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.9086702,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","EoS":true} {"level":"info","ts":1789792478.908721,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","header":"x-maas-subscription"} {"level":"info","ts":1789792478.9087315,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","header":"x-maas-keyname"} {"level":"info","ts":1789792478.9087367,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","header":"x-maas-group"} {"level":"info","ts":1789792478.9087415,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","header":"x-maas-username"} {"level":"info","ts":1789792478.9087565,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789792478.908783,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.9095643,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.9095824,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.9096003,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","EoS":false} {"level":"info","ts":1789792478.9096143,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.909636,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336","EoS":true} {"level":"info","ts":1789792478.9096444,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e21fcd37ac6da21785cc1950a449c9bb","span_id":"b88aa82cb58656bc","x-request-id":"22522025-f182-45d8-b6bd-c5e4aeaec336"} {"level":"info","ts":1789792478.9458554,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789792478.945967,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"info","ts":1789792478.9459825,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"info","ts":1789792478.945995,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","EoS":true} {"level":"info","ts":1789792478.9460163,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","header":"x-maas-keyname"} {"level":"info","ts":1789792478.9460263,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","header":"x-maas-subscription"} {"level":"info","ts":1789792478.9460313,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","header":"x-maas-username"} {"level":"info","ts":1789792478.9460356,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","header":"x-maas-group"} {"level":"info","ts":1789792478.9460497,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789792478.9460766,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"info","ts":1789792478.9467542,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"info","ts":1789792478.946775,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"info","ts":1789792478.946788,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","EoS":false} {"level":"info","ts":1789792478.946804,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"info","ts":1789792478.9468296,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177","EoS":true} {"level":"info","ts":1789792478.9468389,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9120fced9d40bcd886a72d8d8c00f067","span_id":"9344fd929b7f4bcb","x-request-id":"50e05c58-6b5b-46ee-b59c-a0420a0ce177"} {"level":"error","ts":1789792490.4985833,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789792497.1441278,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789792497.1442091,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"92d19129b3b8d052a925e4ee83f6d46c","span_id":"8e1297a68f614f22","x-request-id":"a77aadc3-05e3-4338-b2f1-9860e23ec559"} {"level":"info","ts":1789792497.144222,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"92d19129b3b8d052a925e4ee83f6d46c","span_id":"8e1297a68f614f22","x-request-id":"a77aadc3-05e3-4338-b2f1-9860e23ec559"} {"level":"info","ts":1789792497.1442323,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"92d19129b3b8d052a925e4ee83f6d46c","span_id":"8e1297a68f614f22","x-request-id":"a77aadc3-05e3-4338-b2f1-9860e23ec559","EoS":true} {"level":"info","ts":1789792497.1442747,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"92d19129b3b8d052a925e4ee83f6d46c","span_id":"8e1297a68f614f22","x-request-id":"a77aadc3-05e3-4338-b2f1-9860e23ec559"} {"level":"info","ts":1789792497.3068292,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789792497.3069239,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"475bd10b4cb071402fae32614423021f","span_id":"6d4fad482a0fb5dd","x-request-id":"d7e007da-0f1b-4e93-ba77-e8e3698f80aa"} {"level":"info","ts":1789792497.3069382,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"475bd10b4cb071402fae32614423021f","span_id":"6d4fad482a0fb5dd","x-request-id":"d7e007da-0f1b-4e93-ba77-e8e3698f80aa"} {"level":"info","ts":1789792497.3069503,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"475bd10b4cb071402fae32614423021f","span_id":"6d4fad482a0fb5dd","x-request-id":"d7e007da-0f1b-4e93-ba77-e8e3698f80aa","EoS":true} {"level":"info","ts":1789792497.3069882,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"475bd10b4cb071402fae32614423021f","span_id":"6d4fad482a0fb5dd","x-request-id":"d7e007da-0f1b-4e93-ba77-e8e3698f80aa"} {"level":"info","ts":1789792497.4692047,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789792497.4692984,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a432de9bfd31db5e33457552fd59b2c3","span_id":"543518bb3a2870bc","x-request-id":"331f85d4-80ef-4e31-bb00-29075994eff9"} {"level":"info","ts":1789792497.469312,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a432de9bfd31db5e33457552fd59b2c3","span_id":"543518bb3a2870bc","x-request-id":"331f85d4-80ef-4e31-bb00-29075994eff9"} {"level":"info","ts":1789792497.4693246,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a432de9bfd31db5e33457552fd59b2c3","span_id":"543518bb3a2870bc","x-request-id":"331f85d4-80ef-4e31-bb00-29075994eff9","EoS":true} {"level":"info","ts":1789792497.469369,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a432de9bfd31db5e33457552fd59b2c3","span_id":"543518bb3a2870bc","x-request-id":"331f85d4-80ef-4e31-bb00-29075994eff9"} {"level":"error","ts":1789792510.4998293,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789793609.152665,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793609.1528034,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.1528184,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.1528373,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","EoS":true} {"level":"info","ts":1789793609.1528738,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","header":"x-maas-group"} {"level":"info","ts":1789793609.1528866,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","header":"x-maas-username"} {"level":"info","ts":1789793609.1528962,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","header":"x-maas-keyname"} {"level":"info","ts":1789793609.152905,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","header":"x-maas-subscription"} {"level":"info","ts":1789793609.1529298,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789793609.1529636,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.1537976,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.1538243,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.1538463,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","EoS":false} {"level":"info","ts":1789793609.1538627,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.1538916,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c","EoS":true} {"level":"info","ts":1789793609.153901,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"2f4cfdeba616da4fda6fce0b24d8c0b2","span_id":"e71dc2fa08ce78ef","x-request-id":"86701345-c726-4708-9e96-c98c4434ae3c"} {"level":"info","ts":1789793609.194053,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793609.194172,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"info","ts":1789793609.194187,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"info","ts":1789793609.1942053,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","EoS":true} {"level":"info","ts":1789793609.1942446,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","header":"x-maas-username"} {"level":"info","ts":1789793609.1942565,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","header":"x-maas-group"} {"level":"info","ts":1789793609.1942663,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","header":"x-maas-keyname"} {"level":"info","ts":1789793609.1942747,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","header":"x-maas-subscription"} {"level":"info","ts":1789793609.1942945,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789793609.194326,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"info","ts":1789793609.195044,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"info","ts":1789793609.1950636,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"info","ts":1789793609.195078,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","EoS":false} {"level":"info","ts":1789793609.1950896,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"info","ts":1789793609.1951127,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d","EoS":true} {"level":"info","ts":1789793609.1951227,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c1fda150f074f8a786d47aab7a3bb68a","span_id":"e877841063521279","x-request-id":"c71cd223-4632-46b0-9813-b285e2f73f6d"} {"level":"error","ts":1789793620.628182,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789793940.9300466,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ffeeffb2-dc42-4338-8b75-c6f462bb93f3"} {"level":"info","ts":1789793940.930078,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"86de47b3-77a7-46da-a607-d9ebb9961db0","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789793940.9301188,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"86de47b3-77a7-46da-a607-d9ebb9961db0","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.9301448,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"86de47b3-77a7-46da-a607-d9ebb9961db0"} {"level":"error","ts":1789793940.9373064,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ffeeffb2-dc42-4338-8b75-c6f462bb93f3","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.9373755,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6f0972a0-feaa-4620-9dd6-3d268f8452c3","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789793940.9374037,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6f0972a0-feaa-4620-9dd6-3d268f8452c3","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.9374335,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6f0972a0-feaa-4620-9dd6-3d268f8452c3"} {"level":"error","ts":1789793940.944867,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ffeeffb2-dc42-4338-8b75-c6f462bb93f3","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.9449263,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"70de4562-41af-41c5-8214-7b9874fc1216"} {"level":"info","ts":1789793940.9449499,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c707aba2-bfd1-4ad6-8e2d-d3dc73ede6a5","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789793940.944983,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c707aba2-bfd1-4ad6-8e2d-d3dc73ede6a5","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.945013,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c707aba2-bfd1-4ad6-8e2d-d3dc73ede6a5"} {"level":"error","ts":1789793940.9449506,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"70de4562-41af-41c5-8214-7b9874fc1216","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793940.9507706,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"70de4562-41af-41c5-8214-7b9874fc1216","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.9508216,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"699a2309-31af-448b-ae66-a148d1002126"} {"level":"error","ts":1789793940.9508379,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"699a2309-31af-448b-ae66-a148d1002126","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793940.9573748,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"699a2309-31af-448b-ae66-a148d1002126","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793940.9776287,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4c623c48-fd4e-45e1-9663-19e5ec25cb8b"} {"level":"error","ts":1789793940.9777243,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4c623c48-fd4e-45e1-9663-19e5ec25cb8b","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793940.9832613,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4c623c48-fd4e-45e1-9663-19e5ec25cb8b","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793941.0234559,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4e9e8472-10b0-4a8b-b4f7-b7b12dc541fa"} {"level":"error","ts":1789793941.0235162,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4e9e8472-10b0-4a8b-b4f7-b7b12dc541fa","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793941.0287511,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4e9e8472-10b0-4a8b-b4f7-b7b12dc541fa","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793941.087354,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ae6818c6-29fc-4a55-a895-d8bb97b73d0c"} {"level":"error","ts":1789793941.0874007,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ae6818c6-29fc-4a55-a895-d8bb97b73d0c","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793941.0873654,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"530d08ba-ed0e-4848-a663-a84e971a1b63","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789793941.0874836,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"530d08ba-ed0e-4848-a663-a84e971a1b63","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793941.0875134,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"530d08ba-ed0e-4848-a663-a84e971a1b63"} {"level":"error","ts":1789793941.0926182,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"ae6818c6-29fc-4a55-a895-d8bb97b73d0c","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793941.253148,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"00b7bd79-e495-470e-8489-edd53afa88fe"} {"level":"error","ts":1789793941.2532027,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"00b7bd79-e495-470e-8489-edd53afa88fe","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793941.25901,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"00b7bd79-e495-470e-8489-edd53afa88fe","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793941.5796716,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"25fa77c1-85b0-47c7-816e-43ed7a070177"} {"level":"error","ts":1789793941.5797431,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"25fa77c1-85b0-47c7-816e-43ed7a070177","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793941.5845215,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"25fa77c1-85b0-47c7-816e-43ed7a070177","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793942.225486,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4edc7e9a-016e-4b6d-baa1-a27bf1727180"} {"level":"error","ts":1789793942.2255342,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4edc7e9a-016e-4b6d-baa1-a27bf1727180","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793942.230464,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4edc7e9a-016e-4b6d-baa1-a27bf1727180","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793943.510779,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"91746a18-64c3-420e-b645-12439095206c"} {"level":"error","ts":1789793943.510838,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"91746a18-64c3-420e-b645-12439095206c","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793943.5164597,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"91746a18-64c3-420e-b645-12439095206c","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793946.0773954,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"47b9622a-feb4-4617-b413-18859a7b7998"} {"level":"error","ts":1789793946.0774443,"caller":"externalmodel/reconciler.go:136","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"47b9622a-feb4-4617-b413-18859a7b7998","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:136\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789793946.0827425,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"47b9622a-feb4-4617-b413-18859a7b7998","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793946.087834,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9ecfb9e9-758b-4379-a1ce-a7ed67da6ccc","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789793946.0878708,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9ecfb9e9-758b-4379-a1ce-a7ed67da6ccc","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789793946.0879111,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9ecfb9e9-758b-4379-a1ce-a7ed67da6ccc"} {"level":"info","ts":1789793946.3883,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793946.3884392,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.3884745,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.3885036,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","EoS":true} {"level":"info","ts":1789793946.3885555,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","header":"x-maas-username"} {"level":"info","ts":1789793946.3885686,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","header":"x-maas-group"} {"level":"info","ts":1789793946.3885775,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","header":"x-maas-subscription"} {"level":"info","ts":1789793946.388586,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","header":"x-maas-keyname"} {"level":"info","ts":1789793946.3886082,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793946.3886437,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.389557,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.3895824,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.3896039,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","EoS":false} {"level":"info","ts":1789793946.3896217,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.3896604,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb","EoS":true} {"level":"info","ts":1789793946.3896763,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bf960edac3d746beed864164e62ac9f5","span_id":"2c208462199007a5","x-request-id":"686ddd69-a768-4b29-bdf8-964b0f84b0bb"} {"level":"info","ts":1789793946.5110676,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793946.5111632,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.5111785,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.5111902,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","EoS":true} {"level":"info","ts":1789793946.511216,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","header":"x-maas-keyname"} {"level":"info","ts":1789793946.5112271,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","header":"x-maas-subscription"} {"level":"info","ts":1789793946.511232,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","header":"x-maas-group"} {"level":"info","ts":1789793946.5112379,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","header":"x-maas-username"} {"level":"info","ts":1789793946.5112517,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793946.5112786,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.512011,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.5120385,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.5120604,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","EoS":false} {"level":"info","ts":1789793946.5120935,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.51212,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd","EoS":true} {"level":"info","ts":1789793946.51213,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5d2a36996f4cbd685db7f28f70498831","span_id":"fa7cc8373d723b9c","x-request-id":"4621bad6-921e-41f8-be92-44622aec64dd"} {"level":"info","ts":1789793946.6262004,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793946.6263,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.6263127,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.6263244,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","EoS":true} {"level":"info","ts":1789793946.6263525,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","header":"x-maas-username"} {"level":"info","ts":1789793946.6263633,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","header":"x-maas-keyname"} {"level":"info","ts":1789793946.6263692,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","header":"x-maas-group"} {"level":"info","ts":1789793946.6263733,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","header":"x-maas-subscription"} {"level":"info","ts":1789793946.6263885,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793946.6264148,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.627153,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.6271725,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.6271844,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","EoS":false} {"level":"info","ts":1789793946.627196,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.627222,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad","EoS":true} {"level":"info","ts":1789793946.627229,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ae707ad9bf7d7f3c6a383777958d419d","span_id":"e24077c477e9ecbc","x-request-id":"99e5c701-6098-4791-b215-b2c6351be7ad"} {"level":"info","ts":1789793946.7653103,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"79439eae-9c47-4c51-85f8-a425c2af7685"} {"level":"info","ts":1789793946.7653546,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5f6567f3-d7dd-4cad-b110-6689481399a7","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789793946.7653823,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"5f6567f3-d7dd-4cad-b110-6689481399a7","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789793960.668262,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789793980.3011036,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793980.301197,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.301209,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.3012207,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","EoS":true} {"level":"info","ts":1789793980.3012445,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","header":"x-maas-keyname"} {"level":"info","ts":1789793980.301254,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","header":"x-maas-group"} {"level":"info","ts":1789793980.30126,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","header":"x-maas-subscription"} {"level":"info","ts":1789793980.3012643,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","header":"x-maas-username"} {"level":"info","ts":1789793980.3012784,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793980.3013024,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.3143194,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.3143446,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.314365,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","EoS":false} {"level":"info","ts":1789793980.3143773,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.3144062,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803","EoS":true} {"level":"info","ts":1789793980.3144183,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"60bc07656bbbb3773becf4adf370ecce","span_id":"766e3eda14abf369","x-request-id":"6619be58-b87a-49b7-b5c5-43d4a3be9803"} {"level":"info","ts":1789793980.4567745,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793980.456861,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.456873,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.456884,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","EoS":true} {"level":"info","ts":1789793980.4569073,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","header":"x-maas-subscription"} {"level":"info","ts":1789793980.456917,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","header":"x-maas-group"} {"level":"info","ts":1789793980.4569216,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","header":"x-maas-keyname"} {"level":"info","ts":1789793980.456926,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","header":"x-maas-username"} {"level":"info","ts":1789793980.45694,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793980.456965,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.4578512,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.4578695,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.45788,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","EoS":false} {"level":"info","ts":1789793980.4578917,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.4579172,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e","EoS":true} {"level":"info","ts":1789793980.4579246,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7ddfab011e6376b47d3c43ad1a4cf71b","span_id":"eb1fbe13fd00ed11","x-request-id":"27053d7e-2a38-4011-bc40-c41c6e6c464e"} {"level":"info","ts":1789793980.599904,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793980.6000133,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.6000295,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.6000433,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","EoS":true} {"level":"info","ts":1789793980.600066,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","header":"x-maas-username"} {"level":"info","ts":1789793980.600076,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","header":"x-maas-group"} {"level":"info","ts":1789793980.6000807,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","header":"x-maas-keyname"} {"level":"info","ts":1789793980.6000855,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","header":"x-maas-subscription"} {"level":"info","ts":1789793980.6001003,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793980.6001265,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.6009789,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.6009936,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.601005,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","EoS":false} {"level":"info","ts":1789793980.6010156,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.6010373,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b","EoS":true} {"level":"info","ts":1789793980.6010501,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"af3e15a38842f7a4c6399affd67a8086","span_id":"5a53dcbc33b604c8","x-request-id":"2d163612-4114-47a0-847f-9fb465fe3a6b"} {"level":"info","ts":1789793980.7359238,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793980.736017,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.7360315,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.7360427,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","EoS":true} {"level":"info","ts":1789793980.7360663,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","header":"x-maas-group"} {"level":"info","ts":1789793980.736076,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","header":"x-maas-username"} {"level":"info","ts":1789793980.7360811,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","header":"x-maas-subscription"} {"level":"info","ts":1789793980.7360861,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","header":"x-maas-keyname"} {"level":"info","ts":1789793980.7361004,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793980.7361248,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.7370079,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.7370272,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.7370427,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","EoS":false} {"level":"info","ts":1789793980.7370596,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.7370806,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6","EoS":true} {"level":"info","ts":1789793980.7370872,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"81c3b2877ef69a0bb83d4a22578381b9","span_id":"c41342777eff3a0a","x-request-id":"06b1c4d7-0ffa-4996-98bf-b359283fa4c6"} {"level":"info","ts":1789793980.8850555,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793980.8851721,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"info","ts":1789793980.8851895,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"info","ts":1789793980.8852024,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","EoS":true} {"level":"info","ts":1789793980.8852284,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","header":"x-maas-group"} {"level":"info","ts":1789793980.885238,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","header":"x-maas-subscription"} {"level":"info","ts":1789793980.8852434,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","header":"x-maas-keyname"} {"level":"info","ts":1789793980.885248,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","header":"x-maas-username"} {"level":"info","ts":1789793980.8852632,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793980.88529,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"info","ts":1789793980.8875403,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"info","ts":1789793980.8875663,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"info","ts":1789793980.8875823,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","EoS":false} {"level":"info","ts":1789793980.8875997,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"info","ts":1789793980.8876348,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81","EoS":true} {"level":"info","ts":1789793980.8876488,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"cc07e96c335cedfcb40c6ec3281b2a76","span_id":"09cd0f8f8bdf8529","x-request-id":"22e3214f-e990-4646-badf-22838e9e8d81"} {"level":"error","ts":1789793990.6707916,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789793992.9602506,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793992.9603531,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793992.9603722,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793992.9603848,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","EoS":true} {"level":"info","ts":1789793992.960409,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","header":"x-maas-username"} {"level":"info","ts":1789793992.9604185,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","header":"x-maas-keyname"} {"level":"info","ts":1789793992.9604232,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","header":"x-maas-subscription"} {"level":"info","ts":1789793992.960428,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","header":"x-maas-group"} {"level":"info","ts":1789793992.9604423,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793992.9604733,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793992.962404,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793992.9624245,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793992.9624407,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","EoS":false} {"level":"info","ts":1789793992.962455,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793992.9624796,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278","EoS":true} {"level":"info","ts":1789793992.9624882,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"62053dc841646c001ce31d5742465418","span_id":"b78f5bf708a644da","x-request-id":"2a9df187-e8a1-408e-8893-33013ed56278"} {"level":"info","ts":1789793993.0659995,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793993.0661047,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4"} {"level":"info","ts":1789793993.0669246,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4"} {"level":"info","ts":1789793993.0669436,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4"} {"level":"info","ts":1789793993.0669596,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4","EoS":false} {"level":"info","ts":1789793993.0669746,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4"} {"level":"info","ts":1789793993.0669951,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4","EoS":true} {"level":"info","ts":1789793993.067007,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"592ffa23eaebbd6c3d9488d4aa455075","span_id":"ffac6ba48176e986","x-request-id":"f27920ef-c90e-4d09-86c1-be39820855b4"} {"level":"info","ts":1789793993.8782814,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793993.8783731,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793993.878387,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793993.8783987,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","EoS":true} {"level":"info","ts":1789793993.8784235,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","header":"x-maas-keyname"} {"level":"info","ts":1789793993.8784366,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","header":"x-maas-group"} {"level":"info","ts":1789793993.8784432,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","header":"x-maas-username"} {"level":"info","ts":1789793993.8784485,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","header":"x-maas-subscription"} {"level":"info","ts":1789793993.878463,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793993.8784873,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793993.879233,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793993.8792522,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793993.8792698,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","EoS":false} {"level":"info","ts":1789793993.8792858,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793993.8793218,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443","EoS":true} {"level":"info","ts":1789793993.8793337,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9b60fb85779f46929911ffaf4a82152d","span_id":"2430bf164c3ce74b","x-request-id":"8ee2ae56-ef4d-4893-a734-a0f719e18443"} {"level":"info","ts":1789793994.8604867,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793994.8605862,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793994.8605988,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793994.8606105,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","EoS":true} {"level":"info","ts":1789793994.8606358,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","header":"x-maas-group"} {"level":"info","ts":1789793994.8606455,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","header":"x-maas-username"} {"level":"info","ts":1789793994.8606513,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","header":"x-maas-subscription"} {"level":"info","ts":1789793994.8606555,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","header":"x-maas-keyname"} {"level":"info","ts":1789793994.8606706,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793994.8607166,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793994.8613656,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793994.861384,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793994.8613997,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","EoS":false} {"level":"info","ts":1789793994.861419,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793994.8614557,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44","EoS":true} {"level":"info","ts":1789793994.8614674,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3310830075207ac3e6aed74d4d1557c4","span_id":"3850aa2528a881b3","x-request-id":"ed22d37b-758d-4636-9b0a-064bb90abf44"} {"level":"info","ts":1789793995.610139,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793995.6102488,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793995.6102664,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793995.610279,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","EoS":true} {"level":"info","ts":1789793995.6103032,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","header":"x-maas-group"} {"level":"info","ts":1789793995.6103132,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","header":"x-maas-username"} {"level":"info","ts":1789793995.610319,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","header":"x-maas-keyname"} {"level":"info","ts":1789793995.6103241,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","header":"x-maas-subscription"} {"level":"info","ts":1789793995.6103399,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793995.610365,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793995.6110868,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793995.611107,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793995.6111207,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","EoS":false} {"level":"info","ts":1789793995.611136,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793995.6111667,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7","EoS":true} {"level":"info","ts":1789793995.6111784,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f1ecfc8372e11167f9e4cc8025dd7979","span_id":"984e632afdb62960","x-request-id":"54185c8e-e781-48e6-b3a5-6a0978088bc7"} {"level":"info","ts":1789793996.7069812,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789793996.7070873,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"info","ts":1789793996.7070997,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"info","ts":1789793996.7071116,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","EoS":true} {"level":"info","ts":1789793996.7071369,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","header":"x-maas-username"} {"level":"info","ts":1789793996.7071466,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","header":"x-maas-subscription"} {"level":"info","ts":1789793996.7071514,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","header":"x-maas-keyname"} {"level":"info","ts":1789793996.7071557,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","header":"x-maas-group"} {"level":"info","ts":1789793996.7071707,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789793996.7071972,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"info","ts":1789793996.7078805,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"info","ts":1789793996.7078998,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"info","ts":1789793996.7079146,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","EoS":false} {"level":"info","ts":1789793996.7079325,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"info","ts":1789793996.707969,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84","EoS":true} {"level":"info","ts":1789793996.7079813,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c17512df59645d4eecd121ee5481ef37","span_id":"7943ac969512f7fb","x-request-id":"519451dc-cfeb-442e-99a2-45a0ea696d84"} {"level":"error","ts":1789794005.6716602,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789794020.6730924,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789794020.8961036,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794020.8962357,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"info","ts":1789794020.8962524,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"info","ts":1789794020.8962686,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","EoS":true} {"level":"info","ts":1789794020.896303,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","header":"x-maas-keyname"} {"level":"info","ts":1789794020.896317,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","header":"x-maas-subscription"} {"level":"info","ts":1789794020.8963246,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","header":"x-maas-group"} {"level":"info","ts":1789794020.8963335,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","header":"x-maas-username"} {"level":"info","ts":1789794020.8963537,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789794020.896395,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"info","ts":1789794020.8972774,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"info","ts":1789794020.8973012,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"info","ts":1789794020.8973215,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","EoS":false} {"level":"info","ts":1789794020.8973517,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"info","ts":1789794020.8973892,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951","EoS":true} {"level":"info","ts":1789794020.8974004,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c228bbff87d86a2a6374872e3a9d476e","span_id":"fdfa9d9a779fb6ae","x-request-id":"6e10c6c1-8dcf-4aa9-964e-a1ebb392b951"} {"level":"error","ts":1789794035.6738358,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789794114.4610188,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794114.4611163,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"info","ts":1789794114.4611287,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"info","ts":1789794114.46114,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","EoS":true} {"level":"info","ts":1789794114.4611645,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","header":"x-maas-subscription"} {"level":"info","ts":1789794114.4611745,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","header":"x-maas-keyname"} {"level":"info","ts":1789794114.461179,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","header":"x-maas-username"} {"level":"info","ts":1789794114.4611838,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","header":"x-maas-group"} {"level":"info","ts":1789794114.4611995,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789794114.4612255,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"info","ts":1789794114.474408,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"info","ts":1789794114.474439,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"info","ts":1789794114.4744563,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","EoS":false} {"level":"info","ts":1789794114.4744694,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"info","ts":1789794114.4744997,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad","EoS":true} {"level":"info","ts":1789794114.4745111,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f7b72cc25f84958b4eb7cd7df0ff7704","span_id":"1f024711289677ae","x-request-id":"d8d924ea-9a0b-4955-b5d6-36f3a21230ad"} {"level":"error","ts":1789794125.6859474,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789794208.7860956,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794208.7862253,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"info","ts":1789794208.7862446,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"info","ts":1789794208.7862625,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","EoS":true} {"level":"info","ts":1789794208.7862973,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","header":"x-maas-subscription"} {"level":"info","ts":1789794208.786308,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","header":"x-maas-username"} {"level":"info","ts":1789794208.786319,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","header":"x-maas-group"} {"level":"info","ts":1789794208.7863262,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","header":"x-maas-keyname"} {"level":"info","ts":1789794208.7863448,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789794208.7863803,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"info","ts":1789794208.7873633,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"info","ts":1789794208.787387,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"info","ts":1789794208.7874084,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","EoS":false} {"level":"info","ts":1789794208.787428,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"info","ts":1789794208.7874644,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22","EoS":true} {"level":"info","ts":1789794208.787477,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e307d38abaa276928c7f9bf39177c8ac","span_id":"336b2048242ddb5a","x-request-id":"ff85967f-d714-490f-be98-909ff43d5f22"} {"level":"error","ts":1789794220.6953812,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789794233.210814,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794233.2109325,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.21095,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.210968,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","EoS":true} {"level":"info","ts":1789794233.2110014,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","header":"x-maas-keyname"} {"level":"info","ts":1789794233.2110155,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","header":"x-maas-username"} {"level":"info","ts":1789794233.2110245,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","header":"x-maas-subscription"} {"level":"info","ts":1789794233.211033,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","header":"x-maas-group"} {"level":"info","ts":1789794233.211055,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789794233.2110925,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.224608,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.2246335,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.2246523,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","EoS":false} {"level":"info","ts":1789794233.2246659,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.2247226,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","EoS":false} {"level":"info","ts":1789794233.2247362,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.2247477,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2","EoS":true} {"level":"info","ts":1789794233.2247543,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"5a1dcccd6c9f159efc0e2ea30219ce7c","span_id":"63c1472f257aa9ac","x-request-id":"6164b635-afe5-48fe-9fee-207ccaccb6f2"} {"level":"info","ts":1789794233.3664703,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794233.3665671,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.3665798,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.3665907,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","EoS":true} {"level":"info","ts":1789794233.3666112,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","header":"x-maas-subscription"} {"level":"info","ts":1789794233.366621,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","header":"x-maas-keyname"} {"level":"info","ts":1789794233.3666265,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","header":"x-maas-group"} {"level":"info","ts":1789794233.3666317,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","header":"x-maas-username"} {"level":"info","ts":1789794233.3666456,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789794233.3666747,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.3677268,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.367748,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.3677714,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","EoS":false} {"level":"info","ts":1789794233.3677924,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.3678358,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","EoS":false} {"level":"info","ts":1789794233.3678482,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.3678675,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b","EoS":true} {"level":"info","ts":1789794233.3678796,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"9907528dfa8269a69f10dbaf77edf323","span_id":"ebc50bf08178ac52","x-request-id":"15e52438-2dce-4cad-b99c-aec100d8774b"} {"level":"info","ts":1789794233.5099502,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794233.5100656,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.5100815,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.5101023,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","EoS":true} {"level":"info","ts":1789794233.5101354,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","header":"x-maas-subscription"} {"level":"info","ts":1789794233.5101469,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","header":"x-maas-keyname"} {"level":"info","ts":1789794233.5101557,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","header":"x-maas-group"} {"level":"info","ts":1789794233.5101643,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","header":"x-maas-username"} {"level":"info","ts":1789794233.5101852,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789794233.5102189,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.511278,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.5113046,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.511325,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","EoS":false} {"level":"info","ts":1789794233.5113509,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.5113819,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","EoS":false} {"level":"info","ts":1789794233.511395,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"info","ts":1789794233.5114064,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d","EoS":true} {"level":"info","ts":1789794233.5114152,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"c6ae7cce385bd61511a1fd38c249d3ad","span_id":"3e1caf33651ded30","x-request-id":"e0ac9a41-6631-4e95-a9c4-5dfca17cf42d"} {"level":"error","ts":1789794245.6984563,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789794252.7422466,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789794252.7423532,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.7423677,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.7423797,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","EoS":true} {"level":"info","ts":1789794252.7424011,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","header":"x-maas-group"} {"level":"info","ts":1789794252.7424114,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","header":"x-maas-username"} {"level":"info","ts":1789794252.7424166,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","header":"x-maas-keyname"} {"level":"info","ts":1789794252.7424207,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","header":"x-maas-subscription"} {"level":"info","ts":1789794252.742436,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789794252.7424626,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.7455354,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.7455544,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.745569,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","EoS":false} {"level":"info","ts":1789794252.745581,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.7456071,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","EoS":false} {"level":"info","ts":1789794252.7456183,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"info","ts":1789794252.74563,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2","EoS":true} {"level":"info","ts":1789794252.7456365,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"daa0cb43b4d0c8c235848a1e2aa8da2e","span_id":"da391747208895b6","x-request-id":"a8551c47-145a-95ee-9b73-a7045b6793f2"} {"level":"error","ts":1789794265.7003312,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}