{"level":"info","ts":1789615926.353547,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","EoS":true} {"level":"info","ts":1789615926.3535774,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","header":"x-maas-group"} {"level":"info","ts":1789615926.35359,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","header":"x-maas-subscription"} {"level":"info","ts":1789615926.3535984,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","header":"x-maas-username"} {"level":"info","ts":1789615926.353606,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","header":"x-maas-keyname"} {"level":"info","ts":1789615926.3536255,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789615926.3536587,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01"} {"level":"info","ts":1789615926.3573356,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01"} {"level":"info","ts":1789615926.357384,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01"} {"level":"info","ts":1789615926.3574162,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","EoS":false} {"level":"info","ts":1789615926.3574378,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01"} {"level":"info","ts":1789615926.357487,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01","EoS":true} {"level":"info","ts":1789615926.3575006,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"7347cbb184098b2bf3157c67b2907b40","span_id":"76f9d4577736315e","x-request-id":"a762449a-8fd7-463c-9065-b46c20a24b01"} {"level":"info","ts":1789615926.3925056,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789615926.3925884,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615926.392603,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615926.392614,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","EoS":true} {"level":"info","ts":1789615926.3926327,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","header":"x-maas-subscription"} {"level":"info","ts":1789615926.3926427,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","header":"x-maas-group"} {"level":"info","ts":1789615926.392649,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","header":"x-maas-username"} {"level":"info","ts":1789615926.3926532,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","header":"x-maas-keyname"} {"level":"info","ts":1789615926.3926663,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789615926.3926916,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615926.393497,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615926.393517,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615926.3935359,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","EoS":false} {"level":"info","ts":1789615926.3935504,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615926.3935852,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5","EoS":true} {"level":"info","ts":1789615926.3935962,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"542aaa5437e6f11c794f9bf7f65f096a","span_id":"1184dc4cc5fefc0b","x-request-id":"61a57179-df35-47f3-a589-4c4c7de4b7e5"} {"level":"info","ts":1789615940.1654956,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789615940.165585,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"32c64cb299944b7548f3a06a4a66903b","span_id":"994b70a6ca2df385","x-request-id":"8a5edb99-2906-4f40-a918-47c0bd99c13f"} {"level":"info","ts":1789615940.1655986,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"32c64cb299944b7548f3a06a4a66903b","span_id":"994b70a6ca2df385","x-request-id":"8a5edb99-2906-4f40-a918-47c0bd99c13f"} {"level":"info","ts":1789615940.1656134,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"32c64cb299944b7548f3a06a4a66903b","span_id":"994b70a6ca2df385","x-request-id":"8a5edb99-2906-4f40-a918-47c0bd99c13f","EoS":true} {"level":"info","ts":1789615940.16566,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"32c64cb299944b7548f3a06a4a66903b","span_id":"994b70a6ca2df385","x-request-id":"8a5edb99-2906-4f40-a918-47c0bd99c13f"} {"level":"info","ts":1789615940.3164105,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789615940.3164954,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4c3c36c6966dac771cee4223bb10b8bc","span_id":"3b00cd11cdfb44fd","x-request-id":"b4a95727-0a64-451f-a505-3430758bb430"} {"level":"info","ts":1789615940.3165076,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4c3c36c6966dac771cee4223bb10b8bc","span_id":"3b00cd11cdfb44fd","x-request-id":"b4a95727-0a64-451f-a505-3430758bb430"} {"level":"info","ts":1789615940.31652,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4c3c36c6966dac771cee4223bb10b8bc","span_id":"3b00cd11cdfb44fd","x-request-id":"b4a95727-0a64-451f-a505-3430758bb430","EoS":true} {"level":"info","ts":1789615940.3165588,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4c3c36c6966dac771cee4223bb10b8bc","span_id":"3b00cd11cdfb44fd","x-request-id":"b4a95727-0a64-451f-a505-3430758bb430"} {"level":"error","ts":1789615940.3631823,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789615940.470881,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789615940.471052,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"22af2b0d7b97b2408c2ecc3ab1f4afac","span_id":"9dce2e49c7b82c08","x-request-id":"00483879-8406-4b01-87a7-bb5a9a5c73eb"} {"level":"info","ts":1789615940.4711056,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"22af2b0d7b97b2408c2ecc3ab1f4afac","span_id":"9dce2e49c7b82c08","x-request-id":"00483879-8406-4b01-87a7-bb5a9a5c73eb"} {"level":"info","ts":1789615940.4711347,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"22af2b0d7b97b2408c2ecc3ab1f4afac","span_id":"9dce2e49c7b82c08","x-request-id":"00483879-8406-4b01-87a7-bb5a9a5c73eb","EoS":true} {"level":"info","ts":1789615940.4712074,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"22af2b0d7b97b2408c2ecc3ab1f4afac","span_id":"9dce2e49c7b82c08","x-request-id":"00483879-8406-4b01-87a7-bb5a9a5c73eb"} {"level":"error","ts":1789615955.3648016,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617006.7304513,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617006.730546,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.7305586,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.7305706,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","EoS":true} {"level":"info","ts":1789617006.7305925,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","header":"x-maas-username"} {"level":"info","ts":1789617006.7306018,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","header":"x-maas-keyname"} {"level":"info","ts":1789617006.7306066,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","header":"x-maas-group"} {"level":"info","ts":1789617006.7306118,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","header":"x-maas-subscription"} {"level":"info","ts":1789617006.7306244,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789617006.7306502,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.7314959,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.7315204,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.7315407,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","EoS":false} {"level":"info","ts":1789617006.7315528,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.7315736,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84","EoS":true} {"level":"info","ts":1789617006.7315829,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"65d24fc27ff3134d753ceab7c1975e53","span_id":"acbb43f36ab86da3","x-request-id":"d49facc7-75c8-479d-98d4-5d052809fe84"} {"level":"info","ts":1789617006.770551,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617006.770645,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"info","ts":1789617006.77066,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"info","ts":1789617006.7706773,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","EoS":true} {"level":"info","ts":1789617006.7707083,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","header":"x-maas-subscription"} {"level":"info","ts":1789617006.7707222,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","header":"x-maas-username"} {"level":"info","ts":1789617006.7707305,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","header":"x-maas-group"} {"level":"info","ts":1789617006.7707386,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","header":"x-maas-keyname"} {"level":"info","ts":1789617006.7707572,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","path":"/llm/facebook-opt-125m-simulated/v1/chat/completions"} {"level":"info","ts":1789617006.7707903,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"info","ts":1789617006.771599,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"info","ts":1789617006.771615,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"info","ts":1789617006.7716258,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","EoS":false} {"level":"info","ts":1789617006.771637,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"info","ts":1789617006.771661,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05","EoS":true} {"level":"info","ts":1789617006.7716749,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"4c36355f567cb0434f6ca1d8de01cb13","span_id":"096565f0bf9b89ed","x-request-id":"2640c763-50f6-4d96-bff6-c8d2c874cb05"} {"level":"error","ts":1789617020.4896312,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617337.0000696,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4b6c95f6-73df-4073-84e5-57d8c1799e1c","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789617337.0001388,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4b6c95f6-73df-4073-84e5-57d8c1799e1c","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.0001652,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"4b6c95f6-73df-4073-84e5-57d8c1799e1c"} {"level":"info","ts":1789617337.000076,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e6586d4e-2999-4e12-b94a-ff80c3f170df"} {"level":"info","ts":1789617337.0072646,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e3ba35fd-d232-473f-ba86-5a5236b1f3f6","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789617337.0072927,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e3ba35fd-d232-473f-ba86-5a5236b1f3f6","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.0073168,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e3ba35fd-d232-473f-ba86-5a5236b1f3f6"} {"level":"error","ts":1789617337.0085814,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e6586d4e-2999-4e12-b94a-ff80c3f170df","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-x-api-key-trigger\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:103\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.0086176,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e6586d4e-2999-4e12-b94a-ff80c3f170df","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.01384,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e6586d4e-2999-4e12-b94a-ff80c3f170df","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-x-api-key-trigger\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.0138855,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e6586d4e-2999-4e12-b94a-ff80c3f170df","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.013932,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6d84a32b-fdab-4643-a082-b5eeac767934"} {"level":"error","ts":1789617337.013955,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6d84a32b-fdab-4643-a082-b5eeac767934","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.014133,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"34839dab-24ca-4273-9e0b-8c88469096ff","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789617337.0141633,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"34839dab-24ca-4273-9e0b-8c88469096ff","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.0141873,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"34839dab-24ca-4273-9e0b-8c88469096ff"} {"level":"error","ts":1789617337.0174506,"caller":"externalmodel/reconciler.go:204","msg":"failed to update ExternalModel status","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6d84a32b-fdab-4643-a082-b5eeac767934","error":"Operation cannot be fulfilled on externalmodels.inference.opendatahub.io \"e2e-x-api-key-trigger\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).setStatus\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:204\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:107\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.017486,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6d84a32b-fdab-4643-a082-b5eeac767934","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.0175352,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9576c80e-cbe8-42b4-a370-7a2cd910e9cd"} {"level":"error","ts":1789617337.0175548,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9576c80e-cbe8-42b4-a370-7a2cd910e9cd","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.021473,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9576c80e-cbe8-42b4-a370-7a2cd910e9cd","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.0416949,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c78159fb-f17f-408b-87b7-ce22cb1244da"} {"level":"error","ts":1789617337.041726,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c78159fb-f17f-408b-87b7-ce22cb1244da","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.0463815,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"c78159fb-f17f-408b-87b7-ce22cb1244da","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.0865214,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6f1e26fa-b339-4493-9c1e-1d3846aec4d9"} {"level":"error","ts":1789617337.0865467,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6f1e26fa-b339-4493-9c1e-1d3846aec4d9","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.0908575,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"6f1e26fa-b339-4493-9c1e-1d3846aec4d9","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.1297386,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e656104c-78e5-4941-9ffb-347242d90dab","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789617337.1297722,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e656104c-78e5-4941-9ffb-347242d90dab","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.12979,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"56238214-0e4e-4c46-b191-58c2333e9dda"} {"level":"info","ts":1789617337.1298008,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"e656104c-78e5-4941-9ffb-347242d90dab"} {"level":"error","ts":1789617337.1298225,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"56238214-0e4e-4c46-b191-58c2333e9dda","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.1343224,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"56238214-0e4e-4c46-b191-58c2333e9dda","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.295441,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9cc242ec-474c-45a8-9715-a35c7d1a17b5"} {"level":"error","ts":1789617337.295487,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9cc242ec-474c-45a8-9715-a35c7d1a17b5","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.2996554,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"9cc242ec-474c-45a8-9715-a35c7d1a17b5","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617337.6201565,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3e23645a-714d-4b06-9819-ae4571e285af"} {"level":"error","ts":1789617337.6201887,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3e23645a-714d-4b06-9819-ae4571e285af","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617337.624574,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"3e23645a-714d-4b06-9819-ae4571e285af","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617338.2653666,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"893714b8-94a3-4358-aeac-05ff049a2d8e"} {"level":"error","ts":1789617338.265418,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"893714b8-94a3-4358-aeac-05ff049a2d8e","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617338.2705429,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"893714b8-94a3-4358-aeac-05ff049a2d8e","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617339.5512998,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"566a1454-eaae-49fd-8175-efa3aad84f3e"} {"level":"error","ts":1789617339.5513449,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"566a1454-eaae-49fd-8175-efa3aad84f3e","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617339.555855,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"566a1454-eaae-49fd-8175-efa3aad84f3e","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617342.1163914,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d8f179f8-b73a-4a63-b48a-59e1863668d3"} {"level":"error","ts":1789617342.1164572,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d8f179f8-b73a-4a63-b48a-59e1863668d3","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617342.1209967,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"d8f179f8-b73a-4a63-b48a-59e1863668d3","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617342.1300333,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"91a5127e-d365-47f7-b797-b9f0f3d46520","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789617342.1300726,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"91a5127e-d365-47f7-b797-b9f0f3d46520","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617342.1300912,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"91a5127e-d365-47f7-b797-b9f0f3d46520"} {"level":"info","ts":1789617347.1308932,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"32843071-a392-4503-882a-849b1e11e6df","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"error","ts":1789617347.1309428,"caller":"model-provider-resolver/external_model_reconciler.go:77","msg":"failed to resolve ref, skipping","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"32843071-a392-4503-882a-849b1e11e6df","provider":"dummy-anthropic","error":"ExternalProvider \"dummy-anthropic\" not yet available in store","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver.(*externalModelReconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/plugins/model-provider-resolver/external_model_reconciler.go:77\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617347.1309655,"caller":"model-provider-resolver/external_model_reconciler.go:84","msg":"no ExternalProvider available for any ref, requeuing","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"32843071-a392-4503-882a-849b1e11e6df"} {"level":"info","ts":1789617347.2421014,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"199b8dfc-93f4-44bf-8227-320b71b1e965"} {"level":"error","ts":1789617347.2421436,"caller":"externalmodel/reconciler.go:135","msg":"ExternalProvider not found, skipping","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"199b8dfc-93f4-44bf-8227-320b71b1e965","provider":"dummy-anthropic","error":"ExternalProvider.inference.opendatahub.io \"dummy-anthropic\" not found","stacktrace":"github.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).reconcileHTTPRoute\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:135\ngithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel.(*Reconciler).Reconcile\n\tgithub.com/opendatahub-io/ai-gateway-payload-processing/pkg/controller/externalmodel/reconciler.go:106\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:222\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:479\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"error","ts":1789617347.2475774,"caller":"controller/controller.go:495","msg":"Reconciler error","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"199b8dfc-93f4-44bf-8227-320b71b1e965","error":"ExternalModel \"e2e-x-api-key-trigger\": no provider refs resolved successfully: ExternalProvider \"dummy-anthropic\" not found in namespace \"llm\"","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:495\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:438\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.23.3/pkg/internal/controller/controller.go:313"} {"level":"info","ts":1789617347.5277417,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617347.5278533,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.5278692,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.5278847,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","EoS":true} {"level":"info","ts":1789617347.5279152,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","header":"x-maas-subscription"} {"level":"info","ts":1789617347.5279276,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","header":"x-maas-username"} {"level":"info","ts":1789617347.5279357,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","header":"x-maas-keyname"} {"level":"info","ts":1789617347.527943,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","header":"x-maas-group"} {"level":"info","ts":1789617347.527961,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617347.527994,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.5289779,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.5290027,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.529025,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","EoS":false} {"level":"info","ts":1789617347.529045,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.5290766,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b","EoS":true} {"level":"info","ts":1789617347.5290878,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0c2b36061b93061446596d86c6630160","span_id":"cce373dbb3bfb417","x-request-id":"140c9bc4-aa38-4bc2-8358-38760ddde77b"} {"level":"info","ts":1789617347.5693126,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617347.569398,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.5694098,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.5694222,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","EoS":true} {"level":"info","ts":1789617347.5694413,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","header":"x-maas-group"} {"level":"info","ts":1789617347.5694504,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","header":"x-maas-subscription"} {"level":"info","ts":1789617347.5694559,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","header":"x-maas-keyname"} {"level":"info","ts":1789617347.5694602,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","header":"x-maas-username"} {"level":"info","ts":1789617347.569473,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617347.5694947,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.570378,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.570404,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.570422,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","EoS":false} {"level":"info","ts":1789617347.570442,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.5704775,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852","EoS":true} {"level":"info","ts":1789617347.5704927,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e2af8bc97d277ac074bf67f355892b59","span_id":"00b1fb3fcbd22e80","x-request-id":"781e63f7-5aed-40ae-a594-bbb765866852"} {"level":"info","ts":1789617347.6180062,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617347.6180944,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.6181066,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.6181192,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","EoS":true} {"level":"info","ts":1789617347.618141,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","header":"x-maas-username"} {"level":"info","ts":1789617347.6181533,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","header":"x-maas-subscription"} {"level":"info","ts":1789617347.618161,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","header":"x-maas-group"} {"level":"info","ts":1789617347.6181674,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","header":"x-maas-keyname"} {"level":"info","ts":1789617347.6181834,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617347.6182241,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.619027,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.619046,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.6190598,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","EoS":false} {"level":"info","ts":1789617347.6190755,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.619106,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0","EoS":true} {"level":"info","ts":1789617347.6191154,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"01090da8992809e7c9a759597fed53f2","span_id":"1fcbe1dacd4144f5","x-request-id":"e7947e02-7491-4e97-8343-64971919f2c0"} {"level":"info","ts":1789617347.7179465,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617347.718037,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.7180536,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.7180672,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","EoS":true} {"level":"info","ts":1789617347.7180886,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","header":"x-maas-subscription"} {"level":"info","ts":1789617347.718096,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","header":"x-maas-keyname"} {"level":"info","ts":1789617347.7181003,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","header":"x-maas-username"} {"level":"info","ts":1789617347.7181046,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","header":"x-maas-group"} {"level":"info","ts":1789617347.7181177,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617347.7181432,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.718927,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.7189422,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.718952,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","EoS":false} {"level":"info","ts":1789617347.7189622,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.7189846,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d","EoS":true} {"level":"info","ts":1789617347.7189946,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"150dd1806beb0b10e099dcefa33e95b9","span_id":"22ee1933888e9504","x-request-id":"ae4d1c2c-240f-4b2e-bc3b-335a255a4b3d"} {"level":"info","ts":1789617347.8601341,"caller":"model-provider-resolver/external_model_reconciler.go:48","msg":"reconciling ExternalModel","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"639b0db2-a0b9-4fcc-bd3e-56b13c0349e6","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789617347.8601778,"caller":"model-provider-resolver/external_model_reconciler.go:63","msg":"ExternalModel removed from store","controller":"inference-externalmodel","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"639b0db2-a0b9-4fcc-bd3e-56b13c0349e6","name":"e2e-x-api-key-trigger","namespace":"llm"} {"level":"info","ts":1789617347.8601868,"caller":"externalmodel/reconciler.go:88","msg":"Reconciling ExternalModel","controller":"external-model-controller","controllerGroup":"inference.opendatahub.io","controllerKind":"ExternalModel","ExternalModel":{"name":"e2e-x-api-key-trigger","namespace":"llm"},"namespace":"llm","name":"e2e-x-api-key-trigger","reconcileID":"018a7e2a-1fec-4bab-ad71-948ad82f05ed"} {"level":"error","ts":1789617360.5287032,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617365.4530866,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617365.4532075,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.4532206,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.4532402,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","EoS":true} {"level":"info","ts":1789617365.4532735,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","header":"x-maas-subscription"} {"level":"info","ts":1789617365.4532895,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","header":"x-maas-keyname"} {"level":"info","ts":1789617365.4532988,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","header":"x-maas-group"} {"level":"info","ts":1789617365.453307,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","header":"x-maas-username"} {"level":"info","ts":1789617365.4533267,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617365.4533622,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.4656386,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.4656596,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.465674,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","EoS":false} {"level":"info","ts":1789617365.465686,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.465711,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0","EoS":true} {"level":"info","ts":1789617365.4657176,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"a2dbceb284a3a1c181422bf4be585998","span_id":"cabd84ea51b7323e","x-request-id":"5db47e03-ba66-40ab-a4dd-0c8a825b72a0"} {"level":"info","ts":1789617365.6026385,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617365.6027074,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.6027203,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.602731,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","EoS":true} {"level":"info","ts":1789617365.6027489,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","header":"x-maas-username"} {"level":"info","ts":1789617365.6027582,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","header":"x-maas-group"} {"level":"info","ts":1789617365.6027641,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","header":"x-maas-subscription"} {"level":"info","ts":1789617365.602768,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","header":"x-maas-keyname"} {"level":"info","ts":1789617365.6027827,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617365.6028028,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.604404,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.6044219,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.6044412,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","EoS":false} {"level":"info","ts":1789617365.6044567,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.60449,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4","EoS":true} {"level":"info","ts":1789617365.6045012,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"f198c5d25d3c49e8b816f6b952622ae2","span_id":"c4764004ec340d3a","x-request-id":"5c2b54f1-c7c1-4631-9878-72d82eb4f5d4"} {"level":"info","ts":1789617365.7449389,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617365.7450294,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.7450428,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.7450602,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","EoS":true} {"level":"info","ts":1789617365.7450867,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","header":"x-maas-username"} {"level":"info","ts":1789617365.7451,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","header":"x-maas-group"} {"level":"info","ts":1789617365.7451084,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","header":"x-maas-subscription"} {"level":"info","ts":1789617365.7451158,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","header":"x-maas-keyname"} {"level":"info","ts":1789617365.7451358,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617365.745169,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.7457747,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.7457922,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.7458057,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","EoS":false} {"level":"info","ts":1789617365.7458196,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.7458508,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8","EoS":true} {"level":"info","ts":1789617365.7458599,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b267914895373948dd615dc3b294f4e0","span_id":"d1d79421dc525c3d","x-request-id":"a3e7abb3-a8d0-450b-809a-f8c816d21fe8"} {"level":"info","ts":1789617365.8791447,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617365.8792472,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617365.8792598,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617365.8792713,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","EoS":true} {"level":"info","ts":1789617365.8792894,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","header":"x-maas-group"} {"level":"info","ts":1789617365.8792977,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","header":"x-maas-subscription"} {"level":"info","ts":1789617365.879303,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","header":"x-maas-username"} {"level":"info","ts":1789617365.8793077,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","header":"x-maas-keyname"} {"level":"info","ts":1789617365.8793185,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617365.8793392,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617365.8799345,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617365.8799517,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617365.8799644,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","EoS":false} {"level":"info","ts":1789617365.8799736,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617365.8799958,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4","EoS":true} {"level":"info","ts":1789617365.8800037,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ac8e6313f3b629728ffea1f3432acae6","span_id":"66e183bb90f2592e","x-request-id":"201d8ce0-34d0-4897-ac30-32d103d82ca4"} {"level":"info","ts":1789617366.023221,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617366.0232916,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"info","ts":1789617366.0233023,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"info","ts":1789617366.0233135,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","EoS":true} {"level":"info","ts":1789617366.0233312,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","header":"x-maas-username"} {"level":"info","ts":1789617366.0233417,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","header":"x-maas-group"} {"level":"info","ts":1789617366.023347,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","header":"x-maas-keyname"} {"level":"info","ts":1789617366.0233526,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","header":"x-maas-subscription"} {"level":"info","ts":1789617366.0233653,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617366.023384,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"info","ts":1789617366.0239675,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"info","ts":1789617366.0239835,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"info","ts":1789617366.0239954,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","EoS":false} {"level":"info","ts":1789617366.024007,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"info","ts":1789617366.0240295,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e","EoS":true} {"level":"info","ts":1789617366.0240376,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b8d5054ed9518d799dc70520a3bd798c","span_id":"1879d636c8e1535f","x-request-id":"86c50bb8-d1df-46bd-87ef-b3d51024307e"} {"level":"error","ts":1789617375.5300825,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617377.9146776,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617377.91477,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617377.9147856,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617377.9148033,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","EoS":true} {"level":"info","ts":1789617377.9148247,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","header":"x-maas-subscription"} {"level":"info","ts":1789617377.9148355,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","header":"x-maas-keyname"} {"level":"info","ts":1789617377.9148402,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","header":"x-maas-username"} {"level":"info","ts":1789617377.9148445,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","header":"x-maas-group"} {"level":"info","ts":1789617377.9148583,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617377.914887,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617377.9162676,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617377.9162862,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617377.9163017,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","EoS":false} {"level":"info","ts":1789617377.9163184,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617377.9163392,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2","EoS":true} {"level":"info","ts":1789617377.916349,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"bf3b35a605199738161bd13ceaf3e03c","span_id":"4db5965111f428ef","x-request-id":"9291dd2b-e18c-467d-b867-071cfea413a2"} {"level":"info","ts":1789617378.0124137,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617378.0124977,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1"} {"level":"info","ts":1789617378.0129852,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1"} {"level":"info","ts":1789617378.0130022,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1"} {"level":"info","ts":1789617378.0130124,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1","EoS":false} {"level":"info","ts":1789617378.013024,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1"} {"level":"info","ts":1789617378.0130472,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1","EoS":true} {"level":"info","ts":1789617378.0130622,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"0f33eecc464acce4633fa37df0902330","span_id":"8ed00714bfcdcab3","x-request-id":"676e7585-499d-46d7-8e6b-6af77a848be1"} {"level":"info","ts":1789617378.7736988,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617378.7738228,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617378.7738395,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617378.7738514,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","EoS":true} {"level":"info","ts":1789617378.7738714,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","header":"x-maas-username"} {"level":"info","ts":1789617378.7738814,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","header":"x-maas-subscription"} {"level":"info","ts":1789617378.7738862,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","header":"x-maas-group"} {"level":"info","ts":1789617378.773891,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","header":"x-maas-keyname"} {"level":"info","ts":1789617378.7739065,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617378.7739344,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617378.7748528,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617378.774876,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617378.7748935,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","EoS":false} {"level":"info","ts":1789617378.7749088,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617378.7749412,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f","EoS":true} {"level":"info","ts":1789617378.7749515,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"82880a4316d7afa041d06ff0402cb0f9","span_id":"edf5eec44e87456d","x-request-id":"9ccf5c2c-5b64-4e67-83c7-5430ec8f7f1f"} {"level":"info","ts":1789617379.690166,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617379.6902828,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617379.690298,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617379.6903107,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","EoS":true} {"level":"info","ts":1789617379.6903336,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","header":"x-maas-group"} {"level":"info","ts":1789617379.6903431,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","header":"x-maas-subscription"} {"level":"info","ts":1789617379.6903484,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","header":"x-maas-username"} {"level":"info","ts":1789617379.6903534,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","header":"x-maas-keyname"} {"level":"info","ts":1789617379.6903675,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617379.6903923,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617379.6912558,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617379.6912756,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617379.6913054,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","EoS":false} {"level":"info","ts":1789617379.6913233,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617379.6913483,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47","EoS":true} {"level":"info","ts":1789617379.6913574,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"dbfb4d376fd87b06e4a7738963f109f5","span_id":"ef7d54f53cd71a76","x-request-id":"6ac4240c-77cf-4407-ab0d-9e3c8fc69c47"} {"level":"info","ts":1789617380.3812823,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617380.3813715,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617380.3813856,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617380.381402,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","EoS":true} {"level":"info","ts":1789617380.3814247,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","header":"x-maas-username"} {"level":"info","ts":1789617380.381435,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","header":"x-maas-keyname"} {"level":"info","ts":1789617380.3814402,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","header":"x-maas-subscription"} {"level":"info","ts":1789617380.381445,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","header":"x-maas-group"} {"level":"info","ts":1789617380.3814595,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617380.381492,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617380.3825057,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617380.3825197,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617380.382531,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","EoS":false} {"level":"info","ts":1789617380.382541,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617380.3825684,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67","EoS":true} {"level":"info","ts":1789617380.3825798,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"e8983d83689d195bbadbf6fc52e76fe0","span_id":"a0099b98ab5639b9","x-request-id":"209bc884-2321-4308-a639-b9e4bc693f67"} {"level":"info","ts":1789617381.4022958,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617381.4023883,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"info","ts":1789617381.4024029,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"info","ts":1789617381.402416,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","EoS":true} {"level":"info","ts":1789617381.4024363,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","header":"x-maas-subscription"} {"level":"info","ts":1789617381.4024463,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","header":"x-maas-group"} {"level":"info","ts":1789617381.4024525,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","header":"x-maas-keyname"} {"level":"info","ts":1789617381.4024572,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","header":"x-maas-username"} {"level":"info","ts":1789617381.4024723,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617381.402502,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"info","ts":1789617381.403385,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"info","ts":1789617381.4034073,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"info","ts":1789617381.4034228,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","EoS":false} {"level":"info","ts":1789617381.4034395,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"info","ts":1789617381.403469,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4","EoS":true} {"level":"info","ts":1789617381.4034784,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"485452fc4020615dd04fcaf6d6f6de6f","span_id":"3e2b0a75766be0ee","x-request-id":"eded041a-86d1-474b-9eba-25e0b21b3dc4"} {"level":"error","ts":1789617390.5321648,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"error","ts":1789617400.5325613,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617402.9143965,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617402.9145052,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"info","ts":1789617402.9145222,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"info","ts":1789617402.9145348,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","EoS":true} {"level":"info","ts":1789617402.9145584,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","header":"x-maas-subscription"} {"level":"info","ts":1789617402.9145687,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","header":"x-maas-keyname"} {"level":"info","ts":1789617402.9145744,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","header":"x-maas-username"} {"level":"info","ts":1789617402.9145794,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","header":"x-maas-group"} {"level":"info","ts":1789617402.914593,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","path":"/llm/facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617402.9146183,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"info","ts":1789617402.9154983,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"info","ts":1789617402.9155219,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"info","ts":1789617402.9155388,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","EoS":false} {"level":"info","ts":1789617402.915551,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"info","ts":1789617402.915575,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601","EoS":true} {"level":"info","ts":1789617402.9155838,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"14f071e7b874eb87cb0d5a92a91b1a93","span_id":"7fd9c5af94797263","x-request-id":"5451ed0e-df39-4852-91ea-024a29ea5601"} {"level":"error","ts":1789617415.5343754,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617498.2476718,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617498.2477937,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"info","ts":1789617498.2478096,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"info","ts":1789617498.2478223,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","EoS":true} {"level":"info","ts":1789617498.247843,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","header":"x-maas-subscription"} {"level":"info","ts":1789617498.2478542,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","header":"x-maas-group"} {"level":"info","ts":1789617498.24786,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","header":"x-maas-username"} {"level":"info","ts":1789617498.247864,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","header":"x-maas-keyname"} {"level":"info","ts":1789617498.2478776,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","path":"/llm/e2e-trlp-test-simulated/v1/completions"} {"level":"info","ts":1789617498.2479033,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"info","ts":1789617498.2600756,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"info","ts":1789617498.2600951,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"info","ts":1789617498.2601147,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","EoS":false} {"level":"info","ts":1789617498.2601295,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"info","ts":1789617498.2601602,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426","EoS":true} {"level":"info","ts":1789617498.2601757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"3623d18412e838abf6691435721ce13a","span_id":"405ce82d14b39708","x-request-id":"ed5adbe5-4578-4b7a-9b06-86850b954426"} {"level":"error","ts":1789617510.5456045,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617578.0313349,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617578.0314322,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"info","ts":1789617578.0314453,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"info","ts":1789617578.0314565,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","EoS":true} {"level":"info","ts":1789617578.0314763,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","header":"x-maas-username"} {"level":"info","ts":1789617578.0314858,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","header":"x-maas-keyname"} {"level":"info","ts":1789617578.0314906,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","header":"x-maas-group"} {"level":"info","ts":1789617578.0314956,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","header":"x-maas-subscription"} {"level":"info","ts":1789617578.031509,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","path":"/llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions"} {"level":"info","ts":1789617578.031533,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"info","ts":1789617578.0321712,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"info","ts":1789617578.0322216,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"info","ts":1789617578.032236,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","EoS":false} {"level":"info","ts":1789617578.0322468,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"info","ts":1789617578.0322652,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72","EoS":true} {"level":"info","ts":1789617578.0322723,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"ac042bd56c0163356904f070db1e9162","span_id":"e598a52d680eb7fa","x-request-id":"38a6992c-18af-4894-9a7f-01481985ae72"} {"level":"error","ts":1789617590.5542655,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617591.5872247,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617591.587332,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.5873492,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.5873625,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","EoS":true} {"level":"info","ts":1789617591.587383,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","header":"x-maas-keyname"} {"level":"info","ts":1789617591.5873952,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","header":"x-maas-subscription"} {"level":"info","ts":1789617591.5874004,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","header":"x-maas-username"} {"level":"info","ts":1789617591.587406,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","header":"x-maas-group"} {"level":"info","ts":1789617591.587424,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789617591.587488,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.600463,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.600485,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.6005044,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","EoS":false} {"level":"info","ts":1789617591.6005192,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.600544,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","EoS":false} {"level":"info","ts":1789617591.600558,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.6005692,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa","EoS":true} {"level":"info","ts":1789617591.6005757,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"916aea9de1702d7d29738d9fe581d87a","span_id":"237d3a25c97c2531","x-request-id":"b5243788-c785-4c3d-88d2-def1c30bc9fa"} {"level":"info","ts":1789617591.7339935,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617591.7340925,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.7341063,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.7341235,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","EoS":true} {"level":"info","ts":1789617591.7341537,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","header":"x-maas-keyname"} {"level":"info","ts":1789617591.734168,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","header":"x-maas-subscription"} {"level":"info","ts":1789617591.7341764,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","header":"x-maas-username"} {"level":"info","ts":1789617591.7341855,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","header":"x-maas-group"} {"level":"info","ts":1789617591.7342272,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789617591.7342608,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.7351077,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.7351239,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.7351391,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","EoS":false} {"level":"info","ts":1789617591.7351506,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.73517,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","EoS":false} {"level":"info","ts":1789617591.7351794,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.7352045,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea","EoS":true} {"level":"info","ts":1789617591.7352138,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"b6adaad2550652b9566669c902a54a8a","span_id":"95c9b6363b83907f","x-request-id":"f6b6a807-5b8f-49a3-9c6a-3dc72fd2b1ea"} {"level":"info","ts":1789617591.872872,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617591.8729606,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.8729749,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.8729918,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","EoS":true} {"level":"info","ts":1789617591.8730202,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","header":"x-maas-username"} {"level":"info","ts":1789617591.8730333,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","header":"x-maas-subscription"} {"level":"info","ts":1789617591.8730419,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","header":"x-maas-group"} {"level":"info","ts":1789617591.8730495,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","header":"x-maas-keyname"} {"level":"info","ts":1789617591.8730662,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789617591.8730972,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.8752913,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.8753068,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.8753164,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","EoS":false} {"level":"info","ts":1789617591.8753285,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.8753529,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","EoS":false} {"level":"info","ts":1789617591.8753643,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"info","ts":1789617591.875376,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e","EoS":true} {"level":"info","ts":1789617591.8753822,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"24af9f11e63672559ae8289da174a657","span_id":"e2bc31dac286c223","x-request-id":"420d8412-ba04-4b2d-8c06-d47e8a7e471e"} {"level":"error","ts":1789617605.5561323,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"} {"level":"info","ts":1789617605.7985356,"caller":"handlers/server.go:147","msg":"Processing"} {"level":"info","ts":1789617605.7986212,"caller":"handlers/request.go:66","msg":"captured request headers, deferring response until body arrives...","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.7986357,"caller":"handlers/server.go:205","msg":"processing request headers complete","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.7986526,"caller":"handlers/server.go:207","msg":"Incoming request body chunk","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","EoS":true} {"level":"info","ts":1789617605.7986796,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","header":"x-maas-group"} {"level":"info","ts":1789617605.7986948,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","header":"x-maas-username"} {"level":"info","ts":1789617605.7987032,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","header":"x-maas-keyname"} {"level":"info","ts":1789617605.7987108,"caller":"maas-headers-guard/plugin.go:72","msg":"internal header captured and stripped","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","header":"x-maas-subscription"} {"level":"info","ts":1789617605.7987292,"caller":"model-provider-resolver/plugin.go:175","msg":"received incoming request","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","path":"/llm/e2e-embedding-simulated/v1/embeddings"} {"level":"info","ts":1789617605.7987607,"caller":"handlers/server.go:218","msg":"processing request body complete","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.8006926,"caller":"handlers/response.go:51","msg":"captured response headers, deferring response until body arrives...","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.8007112,"caller":"handlers/server.go:223","msg":"processing response headers complete","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.8007271,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","EoS":false} {"level":"info","ts":1789617605.8007395,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.8007655,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","EoS":false} {"level":"info","ts":1789617605.800784,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"info","ts":1789617605.8007963,"caller":"handlers/server.go:225","msg":"Incoming response body chunk","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978","EoS":true} {"level":"info","ts":1789617605.8008049,"caller":"handlers/server.go:246","msg":"response chunk processing complete","trace_id":"490e98a2acf7671186a927a5f5434a67","span_id":"3ffeaa4145a48aeb","x-request-id":"f66bb7ea-53c1-469b-9731-559c7d52c978"} {"level":"error","ts":1789617620.5574253,"logger":"setup.trace","caller":"tracing/telemetry.go:43","msg":"trace error occurred","error":"traces export: exporter export timeout: rpc error: code = Unavailable desc = name resolver error: produced zero addresses","stacktrace":"github.com/llm-d/llm-d-inference-payload-processor/pkg/common/observability/tracing.(*errorHandler).Handle\n\tgithub.com/llm-d/llm-d-inference-payload-processor@v0.1.0-rc.4.0.20260814090434-bf71acdf306a/pkg/common/observability/tracing/telemetry.go:43\ngo.opentelemetry.io/otel.Handle\n\tgo.opentelemetry.io/otel@v1.45.0/handler.go:33\ngo.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:333\ngo.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func2\n\tgo.opentelemetry.io/otel/sdk@v1.45.0/trace/batch_span_processor.go:128\nsync.(*WaitGroup).Go.func1\n\tsync/waitgroup.go:258"}